132 	struct stm32_qspi *qspi = (struct stm32_qspi *)dev_id;  in stm32_qspi_irq()  local
135 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
136 	sr = readl_relaxed(qspi->io_base + QSPI_SR);  in stm32_qspi_irq()
141 		writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
142 		complete(&qspi->match_completion);  in stm32_qspi_irq()
150 		writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_irq()
151 		complete(&qspi->data_completion);  in stm32_qspi_irq()
167 static int stm32_qspi_tx_poll(struct stm32_qspi *qspi,  in stm32_qspi_tx_poll()  argument
185 		ret = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR,  in stm32_qspi_tx_poll()
189 			dev_err(qspi->dev, "fifo timeout (len:%d stat:%#x)\n",  in stm32_qspi_tx_poll()
193 		tx_fifo(buf++, qspi->io_base + QSPI_DR);  in stm32_qspi_tx_poll()
199 static int stm32_qspi_tx_mm(struct stm32_qspi *qspi,  in stm32_qspi_tx_mm()  argument
202 	memcpy_fromio(op->data.buf.in, qspi->mm_base + op->addr.val,  in stm32_qspi_tx_mm()
214 static int stm32_qspi_tx_dma(struct stm32_qspi *qspi,  in stm32_qspi_tx_dma()  argument
227 		dma_ch = qspi->dma_chrx;  in stm32_qspi_tx_dma()
230 		dma_ch = qspi->dma_chtx;  in stm32_qspi_tx_dma()
237 	err = spi_controller_dma_map_mem_op_data(qspi->ctrl, op, &sgt);  in stm32_qspi_tx_dma()
248 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
250 	reinit_completion(&qspi->dma_completion);  in stm32_qspi_tx_dma()
252 	desc->callback_param = &qspi->dma_completion;  in stm32_qspi_tx_dma()
260 	writel_relaxed(cr | CR_DMAEN, qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
263 	if (!wait_for_completion_timeout(&qspi->dma_completion,  in stm32_qspi_tx_dma()
271 	writel_relaxed(cr & ~CR_DMAEN, qspi->io_base + QSPI_CR);  in stm32_qspi_tx_dma()
273 	spi_controller_dma_unmap_mem_op_data(qspi->ctrl, op, &sgt);  in stm32_qspi_tx_dma()
278 static int stm32_qspi_tx(struct stm32_qspi *qspi, const struct spi_mem_op *op)  in stm32_qspi_tx()  argument
283 	if (qspi->fmode == CCR_FMODE_MM)  in stm32_qspi_tx()
284 		return stm32_qspi_tx_mm(qspi, op);  in stm32_qspi_tx()
285 	else if (((op->data.dir == SPI_MEM_DATA_IN && qspi->dma_chrx) ||  in stm32_qspi_tx()
286 		 (op->data.dir == SPI_MEM_DATA_OUT && qspi->dma_chtx)) &&  in stm32_qspi_tx()
288 		if (!stm32_qspi_tx_dma(qspi, op))  in stm32_qspi_tx()
291 	return stm32_qspi_tx_poll(qspi, op);  in stm32_qspi_tx()
294 static int stm32_qspi_wait_nobusy(struct stm32_qspi *qspi)  in stm32_qspi_wait_nobusy()  argument
298 	return readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_SR, sr,  in stm32_qspi_wait_nobusy()
303 static int stm32_qspi_wait_cmd(struct stm32_qspi *qspi)  in stm32_qspi_wait_cmd()  argument
308 	if ((readl_relaxed(qspi->io_base + QSPI_SR) & SR_TCF) ||  in stm32_qspi_wait_cmd()
309 	    qspi->fmode == CCR_FMODE_APM)  in stm32_qspi_wait_cmd()
312 	reinit_completion(&qspi->data_completion);  in stm32_qspi_wait_cmd()
313 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_wait_cmd()
314 	writel_relaxed(cr | CR_TCIE | CR_TEIE, qspi->io_base + QSPI_CR);  in stm32_qspi_wait_cmd()
316 	if (!wait_for_completion_timeout(&qspi->data_completion,  in stm32_qspi_wait_cmd()
320 		sr = readl_relaxed(qspi->io_base + QSPI_SR);  in stm32_qspi_wait_cmd()
327 	writel_relaxed(FCR_CTCF | FCR_CTEF, qspi->io_base + QSPI_FCR);  in stm32_qspi_wait_cmd()
329 		err = stm32_qspi_wait_nobusy(qspi);  in stm32_qspi_wait_cmd()
334 static int stm32_qspi_wait_poll_status(struct stm32_qspi *qspi)  in stm32_qspi_wait_poll_status()  argument
338 	reinit_completion(&qspi->match_completion);  in stm32_qspi_wait_poll_status()
339 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_wait_poll_status()
340 	writel_relaxed(cr | CR_SMIE, qspi->io_base + QSPI_CR);  in stm32_qspi_wait_poll_status()
342 	if (!wait_for_completion_timeout(&qspi->match_completion,  in stm32_qspi_wait_poll_status()
343 				msecs_to_jiffies(qspi->status_timeout)))  in stm32_qspi_wait_poll_status()
346 	writel_relaxed(FCR_CSMF, qspi->io_base + QSPI_FCR);  in stm32_qspi_wait_poll_status()
361 	struct stm32_qspi *qspi = spi_controller_get_devdata(spi->master);  in stm32_qspi_send()  local
362 	struct stm32_qspi_flash *flash = &qspi->flash[spi->chip_select];  in stm32_qspi_send()
366 	dev_dbg(qspi->dev, "cmd:%#x mode:%d.%d.%d.%d addr:%#llx len:%#x\n",  in stm32_qspi_send()
371 	cr = readl_relaxed(qspi->io_base + QSPI_CR);  in stm32_qspi_send()
375 	writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_send()
379 			       qspi->io_base + QSPI_DLR);  in stm32_qspi_send()
381 	ccr = qspi->fmode;  in stm32_qspi_send()
401 	writel_relaxed(ccr, qspi->io_base + QSPI_CCR);  in stm32_qspi_send()
403 	if (op->addr.nbytes && qspi->fmode != CCR_FMODE_MM)  in stm32_qspi_send()
404 		writel_relaxed(op->addr.val, qspi->io_base + QSPI_AR);  in stm32_qspi_send()
406 	if (qspi->fmode == CCR_FMODE_APM)  in stm32_qspi_send()
407 		err_poll_status = stm32_qspi_wait_poll_status(qspi);  in stm32_qspi_send()
409 	err = stm32_qspi_tx(qspi, op);  in stm32_qspi_send()
418 	if (err || err_poll_status || qspi->fmode == CCR_FMODE_MM)  in stm32_qspi_send()
422 	err = stm32_qspi_wait_cmd(qspi);  in stm32_qspi_send()
429 	cr = readl_relaxed(qspi->io_base + QSPI_CR) | CR_ABORT;  in stm32_qspi_send()
430 	writel_relaxed(cr, qspi->io_base + QSPI_CR);  in stm32_qspi_send()
433 	timeout = readl_relaxed_poll_timeout_atomic(qspi->io_base + QSPI_CR,  in stm32_qspi_send()
437 	writel_relaxed(FCR_CTCF | FCR_CSMF, qspi->io_base + QSPI_FCR);  in stm32_qspi_send()
440 		dev_err(qspi->dev, "%s err:%d err_poll_status:%d abort timeout:%d\n",  in stm32_qspi_send()
452 	struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master);  in stm32_qspi_poll_status()  local
458 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_poll_status()
462 	mutex_lock(&qspi->lock);  in stm32_qspi_poll_status()
464 	writel_relaxed(mask, qspi->io_base + QSPI_PSMKR);  in stm32_qspi_poll_status()
465 	writel_relaxed(match, qspi->io_base + QSPI_PSMAR);  in stm32_qspi_poll_status()
466 	qspi->fmode = CCR_FMODE_APM;  in stm32_qspi_poll_status()
467 	qspi->status_timeout = timeout_ms;  in stm32_qspi_poll_status()
470 	mutex_unlock(&qspi->lock);  in stm32_qspi_poll_status()
472 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_poll_status()
473 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_poll_status()
480 	struct stm32_qspi *qspi = spi_controller_get_devdata(mem->spi->master);  in stm32_qspi_exec_op()  local
483 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_exec_op()
487 	mutex_lock(&qspi->lock);  in stm32_qspi_exec_op()
489 		qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_exec_op()
491 		qspi->fmode = CCR_FMODE_INDW;  in stm32_qspi_exec_op()
494 	mutex_unlock(&qspi->lock);  in stm32_qspi_exec_op()
496 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_exec_op()
497 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_exec_op()
504 	struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->master);  in stm32_qspi_dirmap_create()  local
510 	if (!qspi->mm_base && desc->info.op_tmpl.data.dir == SPI_MEM_DATA_IN)  in stm32_qspi_dirmap_create()
513 	if (!qspi->mm_size)  in stm32_qspi_dirmap_create()
522 	struct stm32_qspi *qspi = spi_controller_get_devdata(desc->mem->spi->master);  in stm32_qspi_dirmap_read()  local
527 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_dirmap_read()
531 	mutex_lock(&qspi->lock);  in stm32_qspi_dirmap_read()
537 	dev_dbg(qspi->dev, "%s len = 0x%zx offs = 0x%llx buf = 0x%p\n", __func__, len, offs, buf);  in stm32_qspi_dirmap_read()
544 	if (addr_max < qspi->mm_size && op.addr.buswidth)  in stm32_qspi_dirmap_read()
545 		qspi->fmode = CCR_FMODE_MM;  in stm32_qspi_dirmap_read()
547 		qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_dirmap_read()
550 	mutex_unlock(&qspi->lock);  in stm32_qspi_dirmap_read()
552 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_dirmap_read()
553 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_dirmap_read()
561 	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_transfer_one_message()  local
570 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_transfer_one_message()
574 	mutex_lock(&qspi->lock);  in stm32_qspi_transfer_one_message()
583 		dev_dbg(qspi->dev, "tx_buf:%p tx_nbits:%d rx_buf:%p rx_nbits:%d len:%d dummy_data:%d\n",  in stm32_qspi_transfer_one_message()
589 		 * QSPI hardware supports dummy bytes transfer.  in stm32_qspi_transfer_one_message()
591 		 * transfer in order to take into account QSPI block constraint  in stm32_qspi_transfer_one_message()
610 			qspi->fmode = CCR_FMODE_INDR;  in stm32_qspi_transfer_one_message()
615 			qspi->fmode = CCR_FMODE_INDW;  in stm32_qspi_transfer_one_message()
631 	mutex_unlock(&qspi->lock);  in stm32_qspi_transfer_one_message()
636 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_transfer_one_message()
637 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_transfer_one_message()
645 	struct stm32_qspi *qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_setup()  local
659 	    gpiod_count(qspi->dev, "cs") == -ENOENT)) {  in stm32_qspi_setup()
660 		dev_err(qspi->dev, "spi-rx-bus-width\\/spi-tx-bus-width\\/cs-gpios\n");  in stm32_qspi_setup()
661 		dev_err(qspi->dev, "configuration not supported\n");  in stm32_qspi_setup()
666 	ret = pm_runtime_resume_and_get(qspi->dev);  in stm32_qspi_setup()
670 	presc = DIV_ROUND_UP(qspi->clk_rate, spi->max_speed_hz) - 1;  in stm32_qspi_setup()
672 	flash = &qspi->flash[spi->chip_select];  in stm32_qspi_setup()
676 	mutex_lock(&qspi->lock);  in stm32_qspi_setup()
677 	qspi->cr_reg = CR_APMS | 3 << CR_FTHRES_SHIFT | CR_SSHIFT | CR_EN;  in stm32_qspi_setup()
684 		qspi->cr_reg |= CR_DFM;  in stm32_qspi_setup()
685 		dev_dbg(qspi->dev, "Dual flash mode enable");  in stm32_qspi_setup()
688 	writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);  in stm32_qspi_setup()
691 	qspi->dcr_reg = DCR_FSIZE_MASK;  in stm32_qspi_setup()
692 	writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);  in stm32_qspi_setup()
693 	mutex_unlock(&qspi->lock);  in stm32_qspi_setup()
695 	pm_runtime_mark_last_busy(qspi->dev);  in stm32_qspi_setup()
696 	pm_runtime_put_autosuspend(qspi->dev);  in stm32_qspi_setup()
701 static int stm32_qspi_dma_setup(struct stm32_qspi *qspi)  in stm32_qspi_dma_setup()  argument
704 	struct device *dev = qspi->dev;  in stm32_qspi_dma_setup()
711 	dma_cfg.src_addr = qspi->phys_base + QSPI_DR;  in stm32_qspi_dma_setup()
712 	dma_cfg.dst_addr = qspi->phys_base + QSPI_DR;  in stm32_qspi_dma_setup()
716 	qspi->dma_chrx = dma_request_chan(dev, "rx");  in stm32_qspi_dma_setup()
717 	if (IS_ERR(qspi->dma_chrx)) {  in stm32_qspi_dma_setup()
718 		ret = PTR_ERR(qspi->dma_chrx);  in stm32_qspi_dma_setup()
719 		qspi->dma_chrx = NULL;  in stm32_qspi_dma_setup()
723 		if (dmaengine_slave_config(qspi->dma_chrx, &dma_cfg)) {  in stm32_qspi_dma_setup()
725 			dma_release_channel(qspi->dma_chrx);  in stm32_qspi_dma_setup()
726 			qspi->dma_chrx = NULL;  in stm32_qspi_dma_setup()
730 	qspi->dma_chtx = dma_request_chan(dev, "tx");  in stm32_qspi_dma_setup()
731 	if (IS_ERR(qspi->dma_chtx)) {  in stm32_qspi_dma_setup()
732 		ret = PTR_ERR(qspi->dma_chtx);  in stm32_qspi_dma_setup()
733 		qspi->dma_chtx = NULL;  in stm32_qspi_dma_setup()
735 		if (dmaengine_slave_config(qspi->dma_chtx, &dma_cfg)) {  in stm32_qspi_dma_setup()
737 			dma_release_channel(qspi->dma_chtx);  in stm32_qspi_dma_setup()
738 			qspi->dma_chtx = NULL;  in stm32_qspi_dma_setup()
743 	init_completion(&qspi->dma_completion);  in stm32_qspi_dma_setup()
751 static void stm32_qspi_dma_free(struct stm32_qspi *qspi)  in stm32_qspi_dma_free()  argument
753 	if (qspi->dma_chtx)  in stm32_qspi_dma_free()
754 		dma_release_channel(qspi->dma_chtx);  in stm32_qspi_dma_free()
755 	if (qspi->dma_chrx)  in stm32_qspi_dma_free()
756 		dma_release_channel(qspi->dma_chrx);  in stm32_qspi_dma_free()
775 	struct stm32_qspi *qspi;  in stm32_qspi_probe()  local
779 	ctrl = devm_spi_alloc_master(dev, sizeof(*qspi));  in stm32_qspi_probe()
783 	qspi = spi_controller_get_devdata(ctrl);  in stm32_qspi_probe()
784 	qspi->ctrl = ctrl;  in stm32_qspi_probe()
786 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "qspi");  in stm32_qspi_probe()
787 	qspi->io_base = devm_ioremap_resource(dev, res);  in stm32_qspi_probe()
788 	if (IS_ERR(qspi->io_base))  in stm32_qspi_probe()
789 		return PTR_ERR(qspi->io_base);  in stm32_qspi_probe()
791 	qspi->phys_base = res->start;  in stm32_qspi_probe()
794 	qspi->mm_base = devm_ioremap_resource(dev, res);  in stm32_qspi_probe()
795 	if (IS_ERR(qspi->mm_base))  in stm32_qspi_probe()
796 		return PTR_ERR(qspi->mm_base);  in stm32_qspi_probe()
798 	qspi->mm_size = resource_size(res);  in stm32_qspi_probe()
799 	if (qspi->mm_size > STM32_QSPI_MAX_MMAP_SZ)  in stm32_qspi_probe()
807 			       dev_name(dev), qspi);  in stm32_qspi_probe()
813 	init_completion(&qspi->data_completion);  in stm32_qspi_probe()
814 	init_completion(&qspi->match_completion);  in stm32_qspi_probe()
816 	qspi->clk = devm_clk_get(dev, NULL);  in stm32_qspi_probe()
817 	if (IS_ERR(qspi->clk))  in stm32_qspi_probe()
818 		return PTR_ERR(qspi->clk);  in stm32_qspi_probe()
820 	qspi->clk_rate = clk_get_rate(qspi->clk);  in stm32_qspi_probe()
821 	if (!qspi->clk_rate)  in stm32_qspi_probe()
824 	ret = clk_prepare_enable(qspi->clk);  in stm32_qspi_probe()
841 	qspi->dev = dev;  in stm32_qspi_probe()
842 	platform_set_drvdata(pdev, qspi);  in stm32_qspi_probe()
843 	ret = stm32_qspi_dma_setup(qspi);  in stm32_qspi_probe()
847 	mutex_init(&qspi->lock);  in stm32_qspi_probe()
875 	pm_runtime_get_sync(qspi->dev);  in stm32_qspi_probe()
876 	/* disable qspi */  in stm32_qspi_probe()
877 	writel_relaxed(0, qspi->io_base + QSPI_CR);  in stm32_qspi_probe()
878 	mutex_destroy(&qspi->lock);  in stm32_qspi_probe()
879 	pm_runtime_put_noidle(qspi->dev);  in stm32_qspi_probe()
880 	pm_runtime_disable(qspi->dev);  in stm32_qspi_probe()
881 	pm_runtime_set_suspended(qspi->dev);  in stm32_qspi_probe()
882 	pm_runtime_dont_use_autosuspend(qspi->dev);  in stm32_qspi_probe()
884 	stm32_qspi_dma_free(qspi);  in stm32_qspi_probe()
886 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_probe()
893 	struct stm32_qspi *qspi = platform_get_drvdata(pdev);  in stm32_qspi_remove()  local
895 	pm_runtime_get_sync(qspi->dev);  in stm32_qspi_remove()
896 	spi_unregister_master(qspi->ctrl);  in stm32_qspi_remove()
897 	/* disable qspi */  in stm32_qspi_remove()
898 	writel_relaxed(0, qspi->io_base + QSPI_CR);  in stm32_qspi_remove()
899 	stm32_qspi_dma_free(qspi);  in stm32_qspi_remove()
900 	mutex_destroy(&qspi->lock);  in stm32_qspi_remove()
901 	pm_runtime_put_noidle(qspi->dev);  in stm32_qspi_remove()
902 	pm_runtime_disable(qspi->dev);  in stm32_qspi_remove()
903 	pm_runtime_set_suspended(qspi->dev);  in stm32_qspi_remove()
904 	pm_runtime_dont_use_autosuspend(qspi->dev);  in stm32_qspi_remove()
905 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_remove()
912 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_runtime_suspend()  local
914 	clk_disable_unprepare(qspi->clk);  in stm32_qspi_runtime_suspend()
921 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_runtime_resume()  local
923 	return clk_prepare_enable(qspi->clk);  in stm32_qspi_runtime_resume()
935 	struct stm32_qspi *qspi = dev_get_drvdata(dev);  in stm32_qspi_resume()  local
948 	writel_relaxed(qspi->cr_reg, qspi->io_base + QSPI_CR);  in stm32_qspi_resume()
949 	writel_relaxed(qspi->dcr_reg, qspi->io_base + QSPI_DCR);  in stm32_qspi_resume()
964 	{.compatible = "st,stm32f469-qspi"},
973 		.name = "stm32-qspi",