480 static void emc_ccfifo_writel(struct tegra_emc *emc, u32 value,  in emc_ccfifo_writel()  argument
483 	writel(value, emc->regs + EMC_CCFIFO_DATA);  in emc_ccfifo_writel()
484 	writel(offset, emc->regs + EMC_CCFIFO_ADDR);  in emc_ccfifo_writel()
487 static void emc_seq_update_timing(struct tegra_emc *emc)  in emc_seq_update_timing()  argument
492 	writel(1, emc->regs + EMC_TIMING_CONTROL);  in emc_seq_update_timing()
495 		value = readl(emc->regs + EMC_STATUS);  in emc_seq_update_timing()
501 	dev_err(emc->dev, "timing update timed out\n");  in emc_seq_update_timing()
504 static void emc_seq_disable_auto_cal(struct tegra_emc *emc)  in emc_seq_disable_auto_cal()  argument
509 	writel(0, emc->regs + EMC_AUTO_CAL_INTERVAL);  in emc_seq_disable_auto_cal()
512 		value = readl(emc->regs + EMC_AUTO_CAL_STATUS);  in emc_seq_disable_auto_cal()
518 	dev_err(emc->dev, "auto cal disable timed out\n");  in emc_seq_disable_auto_cal()
521 static void emc_seq_wait_clkchange(struct tegra_emc *emc)  in emc_seq_wait_clkchange()  argument
527 		value = readl(emc->regs + EMC_INTSTATUS);  in emc_seq_wait_clkchange()
533 	dev_err(emc->dev, "clock change timed out\n");  in emc_seq_wait_clkchange()
536 static struct emc_timing *tegra_emc_find_timing(struct tegra_emc *emc,  in tegra_emc_find_timing()  argument
542 	for (i = 0; i < emc->num_timings; i++) {  in tegra_emc_find_timing()
543 		if (emc->timings[i].rate == rate) {  in tegra_emc_find_timing()
544 			timing = &emc->timings[i];  in tegra_emc_find_timing()
550 		dev_err(emc->dev, "no timing for rate %lu\n", rate);  in tegra_emc_find_timing()
557 int tegra_emc_prepare_timing_change(struct tegra_emc *emc,  in tegra_emc_prepare_timing_change()  argument
560 	struct emc_timing *timing = tegra_emc_find_timing(emc, rate);  in tegra_emc_prepare_timing_change()
561 	struct emc_timing *last = &emc->last_timing;  in tegra_emc_prepare_timing_change()
579 	writel(EMC_INTSTATUS_CLKCHANGE_COMPLETE, emc->regs + EMC_INTSTATUS);  in tegra_emc_prepare_timing_change()
582 	val = readl(emc->regs + EMC_CFG);  in tegra_emc_prepare_timing_change()
585 		writel(val, emc->regs + EMC_CFG);  in tegra_emc_prepare_timing_change()
591 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
596 	val = readl(emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_prepare_timing_change()
599 		writel(val, emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_prepare_timing_change()
603 	val = readl(emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_prepare_timing_change()
618 		writel(val2, emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_prepare_timing_change()
624 	val = readl(emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
638 		writel(val, emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
645 		emc_seq_update_timing(emc);  in tegra_emc_prepare_timing_change()
651 		emc_seq_disable_auto_cal(emc);  in tegra_emc_prepare_timing_change()
653 		       emc->regs + EMC_CTT_TERM_CTRL);  in tegra_emc_prepare_timing_change()
654 		emc_seq_update_timing(emc);  in tegra_emc_prepare_timing_change()
660 		       emc->regs + emc_burst_regs[i]);  in tegra_emc_prepare_timing_change()
662 	writel(timing->emc_xm2dqspadctrl2, emc->regs + EMC_XM2DQSPADCTRL2);  in tegra_emc_prepare_timing_change()
663 	writel(timing->emc_zcal_interval, emc->regs + EMC_ZCAL_INTERVAL);  in tegra_emc_prepare_timing_change()
665 	tegra_mc_write_emem_configuration(emc->mc, timing->rate);  in tegra_emc_prepare_timing_change()
668 	emc_ccfifo_writel(emc, val, EMC_CFG);  in tegra_emc_prepare_timing_change()
672 		emc_ccfifo_writel(emc, timing->emc_auto_cal_config2,  in tegra_emc_prepare_timing_change()
676 		emc_ccfifo_writel(emc, timing->emc_auto_cal_config3,  in tegra_emc_prepare_timing_change()
682 		emc_ccfifo_writel(emc, val, EMC_AUTO_CAL_CONFIG);  in tegra_emc_prepare_timing_change()
686 	if (emc->dram_type == DRAM_TYPE_DDR3 &&  in tegra_emc_prepare_timing_change()
692 			cnt -= emc->dram_num * 256;  in tegra_emc_prepare_timing_change()
705 		writel(val, emc->regs + EMC_MRS_WAIT_CNT);  in tegra_emc_prepare_timing_change()
710 	emc_ccfifo_writel(emc, val, EMC_CFG_2);  in tegra_emc_prepare_timing_change()
713 	if (emc->dram_type == DRAM_TYPE_DDR3 && dll_change == DLL_CHANGE_OFF)  in tegra_emc_prepare_timing_change()
714 		emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);  in tegra_emc_prepare_timing_change()
717 	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num),  in tegra_emc_prepare_timing_change()
719 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
720 		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num) |  in tegra_emc_prepare_timing_change()
725 	emc_ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE);  in tegra_emc_prepare_timing_change()
728 	if (emc->dram_type == DRAM_TYPE_DDR3)  in tegra_emc_prepare_timing_change()
729 		emc_ccfifo_writel(emc, EMC_DRAM_DEV_SEL(emc->dram_num),  in tegra_emc_prepare_timing_change()
731 	emc_ccfifo_writel(emc, EMC_REFCTRL_DEV_SEL(emc->dram_num) |  in tegra_emc_prepare_timing_change()
736 	if (emc->dram_type == DRAM_TYPE_DDR3) {  in tegra_emc_prepare_timing_change()
738 			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_EMRS);  in tegra_emc_prepare_timing_change()
740 			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_EMRS2);  in tegra_emc_prepare_timing_change()
751 			emc_ccfifo_writel(emc, val, EMC_MRS);  in tegra_emc_prepare_timing_change()
755 			emc_ccfifo_writel(emc, timing->emc_mode_2, EMC_MRW2);  in tegra_emc_prepare_timing_change()
757 			emc_ccfifo_writel(emc, timing->emc_mode_1, EMC_MRW);  in tegra_emc_prepare_timing_change()
759 			emc_ccfifo_writel(emc, timing->emc_mode_4, EMC_MRW4);  in tegra_emc_prepare_timing_change()
764 		emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV0, EMC_ZQ_CAL);  in tegra_emc_prepare_timing_change()
765 		if (emc->dram_num > 1)  in tegra_emc_prepare_timing_change()
766 			emc_ccfifo_writel(emc, EMC_ZQ_CAL_LONG_CMD_DEV1,  in tegra_emc_prepare_timing_change()
771 	emc_ccfifo_writel(emc, 0, EMC_CCFIFO_STATUS);  in tegra_emc_prepare_timing_change()
774 		emc_ccfifo_writel(emc, timing->emc_cfg_2, EMC_CFG_2);  in tegra_emc_prepare_timing_change()
777 	emc_seq_disable_auto_cal(emc);  in tegra_emc_prepare_timing_change()
780 	readl(emc->regs + EMC_INTSTATUS);  in tegra_emc_prepare_timing_change()
785 void tegra_emc_complete_timing_change(struct tegra_emc *emc,  in tegra_emc_complete_timing_change()  argument
788 	struct emc_timing *timing = tegra_emc_find_timing(emc, rate);  in tegra_emc_complete_timing_change()
789 	struct emc_timing *last = &emc->last_timing;  in tegra_emc_complete_timing_change()
796 	emc_seq_wait_clkchange(emc);  in tegra_emc_complete_timing_change()
801 		       emc->regs + EMC_AUTO_CAL_INTERVAL);  in tegra_emc_complete_timing_change()
805 		writel(timing->emc_cfg, emc->regs + EMC_CFG);  in tegra_emc_complete_timing_change()
808 	writel(timing->emc_zcal_cnt_long, emc->regs + EMC_ZCAL_WAIT_CNT);  in tegra_emc_complete_timing_change()
811 	if (emc->dram_type == DRAM_TYPE_LPDDR3 &&  in tegra_emc_complete_timing_change()
817 		writel(val, emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_complete_timing_change()
819 		if (emc->dram_type == DRAM_TYPE_DDR3 &&  in tegra_emc_complete_timing_change()
820 		    readl(emc->regs + EMC_BGBIAS_CTL0) !=  in tegra_emc_complete_timing_change()
823 			       emc->regs + EMC_BGBIAS_CTL0);  in tegra_emc_complete_timing_change()
827 		       emc->regs + EMC_AUTO_CAL_INTERVAL);  in tegra_emc_complete_timing_change()
834 	writel(timing->emc_sel_dpd_ctrl, emc->regs + EMC_SEL_DPD_CTRL);  in tegra_emc_complete_timing_change()
835 	emc_seq_update_timing(emc);  in tegra_emc_complete_timing_change()
837 	emc->last_timing = *timing;  in tegra_emc_complete_timing_change()
842 static void emc_read_current_timing(struct tegra_emc *emc,  in emc_read_current_timing()  argument
849 			readl(emc->regs + emc_burst_regs[i]);  in emc_read_current_timing()
851 	timing->emc_cfg = readl(emc->regs + EMC_CFG);  in emc_read_current_timing()
861 static int emc_init(struct tegra_emc *emc)  in emc_init()  argument
863 	emc->dram_type = readl(emc->regs + EMC_FBIO_CFG5);  in emc_init()
864 	emc->dram_type &= EMC_FBIO_CFG5_DRAM_TYPE_MASK;  in emc_init()
865 	emc->dram_type >>= EMC_FBIO_CFG5_DRAM_TYPE_SHIFT;  in emc_init()
867 	emc->dram_num = tegra_mc_get_emem_device_count(emc->mc);  in emc_init()
869 	emc_read_current_timing(emc, &emc->last_timing);  in emc_init()
874 static int load_one_timing_from_dt(struct tegra_emc *emc,  in load_one_timing_from_dt()  argument
883 		dev_err(emc->dev, "timing %pOFn: failed to read rate: %d\n",  in load_one_timing_from_dt()
894 		dev_err(emc->dev,  in load_one_timing_from_dt()
903 		dev_err(emc->dev, "timing %pOFn: failed to read " #prop ": %d\n", \  in load_one_timing_from_dt()
945 static int tegra_emc_load_timings_from_dt(struct tegra_emc *emc,  in tegra_emc_load_timings_from_dt()  argument
954 	emc->timings = devm_kcalloc(emc->dev, child_count, sizeof(*timing),  in tegra_emc_load_timings_from_dt()
956 	if (!emc->timings)  in tegra_emc_load_timings_from_dt()
959 	emc->num_timings = child_count;  in tegra_emc_load_timings_from_dt()
962 		timing = &emc->timings[i++];  in tegra_emc_load_timings_from_dt()
964 		err = load_one_timing_from_dt(emc, timing, child);  in tegra_emc_load_timings_from_dt()
971 	sort(emc->timings, emc->num_timings, sizeof(*timing), cmp_timings,  in tegra_emc_load_timings_from_dt()
1024 	struct tegra_emc *emc = s->private;  in emc_debug_supported_rates_show()  local
1028 	for (i = 0; i < emc->num_timings; i++) {  in emc_debug_supported_rates_show()
1029 		struct emc_timing *timing = &emc->timings[i];  in emc_debug_supported_rates_show()
1055 static void emc_debugfs_init(struct device *dev, struct tegra_emc *emc)  in emc_debugfs_init()  argument
1077 	file = debugfs_create_file("supported_rates", S_IRUGO, root, emc,  in emc_debugfs_init()
1087 	struct tegra_emc *emc;  in tegra_emc_probe()  local
1092 	emc = devm_kzalloc(&pdev->dev, sizeof(*emc), GFP_KERNEL);  in tegra_emc_probe()
1093 	if (!emc)  in tegra_emc_probe()
1096 	emc->dev = &pdev->dev;  in tegra_emc_probe()
1099 	emc->regs = devm_ioremap_resource(&pdev->dev, res);  in tegra_emc_probe()
1100 	if (IS_ERR(emc->regs))  in tegra_emc_probe()
1101 		return PTR_ERR(emc->regs);  in tegra_emc_probe()
1114 	emc->mc = platform_get_drvdata(mc);  in tegra_emc_probe()
1115 	if (!emc->mc)  in tegra_emc_probe()
1128 	err = tegra_emc_load_timings_from_dt(emc, np);  in tegra_emc_probe()
1133 	if (emc->num_timings == 0) {  in tegra_emc_probe()
1140 	err = emc_init(emc);  in tegra_emc_probe()
1146 	platform_set_drvdata(pdev, emc);  in tegra_emc_probe()
1149 		emc_debugfs_init(&pdev->dev, emc);  in tegra_emc_probe()