150 static u32 qspi_buswidth_to_iomode(struct qcom_qspi *ctrl,  in qspi_buswidth_to_iomode()  argument
161 		dev_warn_once(ctrl->dev,  in qspi_buswidth_to_iomode()
167 static void qcom_qspi_pio_xfer_cfg(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer_cfg()  argument
172 	xfer = &ctrl->xfer;  in qcom_qspi_pio_xfer_cfg()
173 	pio_xfer_cfg = readl(ctrl->base + PIO_XFER_CFG);  in qcom_qspi_pio_xfer_cfg()
181 	pio_xfer_cfg |= qspi_buswidth_to_iomode(ctrl, xfer->buswidth);  in qcom_qspi_pio_xfer_cfg()
183 	writel(pio_xfer_cfg, ctrl->base + PIO_XFER_CFG);  in qcom_qspi_pio_xfer_cfg()
186 static void qcom_qspi_pio_xfer_ctrl(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer_ctrl()  argument
190 	pio_xfer_ctrl = readl(ctrl->base + PIO_XFER_CTRL);  in qcom_qspi_pio_xfer_ctrl()
192 	pio_xfer_ctrl |= ctrl->xfer.rem_bytes;  in qcom_qspi_pio_xfer_ctrl()
193 	writel(pio_xfer_ctrl, ctrl->base + PIO_XFER_CTRL);  in qcom_qspi_pio_xfer_ctrl()
196 static void qcom_qspi_pio_xfer(struct qcom_qspi *ctrl)  in qcom_qspi_pio_xfer()  argument
200 	qcom_qspi_pio_xfer_cfg(ctrl);  in qcom_qspi_pio_xfer()
203 	writel(QSPI_ALL_IRQS, ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_pio_xfer()
206 	if (ctrl->xfer.dir == QSPI_WRITE)  in qcom_qspi_pio_xfer()
210 	writel(ints, ctrl->base + MSTR_INT_EN);  in qcom_qspi_pio_xfer()
213 	qcom_qspi_pio_xfer_ctrl(ctrl);  in qcom_qspi_pio_xfer()
219 	struct qcom_qspi *ctrl = spi_master_get_devdata(master);  in qcom_qspi_handle_err()  local
222 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_handle_err()
223 	writel(0, ctrl->base + MSTR_INT_EN);  in qcom_qspi_handle_err()
224 	ctrl->xfer.rem_bytes = 0;  in qcom_qspi_handle_err()
225 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_handle_err()
228 static int qcom_qspi_set_speed(struct qcom_qspi *ctrl, unsigned long speed_hz)  in qcom_qspi_set_speed()  argument
233 	if (speed_hz == ctrl->last_speed)  in qcom_qspi_set_speed()
237 	ret = dev_pm_opp_set_rate(ctrl->dev, speed_hz * 4);  in qcom_qspi_set_speed()
239 		dev_err(ctrl->dev, "Failed to set core clk %d\n", ret);  in qcom_qspi_set_speed()
248 	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, avg_bw_cpu, avg_bw_cpu);  in qcom_qspi_set_speed()
250 		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",  in qcom_qspi_set_speed()
255 	ctrl->last_speed = speed_hz;  in qcom_qspi_set_speed()
264 	struct qcom_qspi *ctrl = spi_master_get_devdata(master);  in qcom_qspi_transfer_one()  local
273 	ret = qcom_qspi_set_speed(ctrl, speed_hz);  in qcom_qspi_transfer_one()
277 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_transfer_one()
281 		ctrl->xfer.dir = QSPI_READ;  in qcom_qspi_transfer_one()
282 		ctrl->xfer.buswidth = xfer->rx_nbits;  in qcom_qspi_transfer_one()
283 		ctrl->xfer.rx_buf = xfer->rx_buf;  in qcom_qspi_transfer_one()
285 		ctrl->xfer.dir = QSPI_WRITE;  in qcom_qspi_transfer_one()
286 		ctrl->xfer.buswidth = xfer->tx_nbits;  in qcom_qspi_transfer_one()
287 		ctrl->xfer.tx_buf = xfer->tx_buf;  in qcom_qspi_transfer_one()
289 	ctrl->xfer.is_last = list_is_last(&xfer->transfer_list,  in qcom_qspi_transfer_one()
291 	ctrl->xfer.rem_bytes = xfer->len;  in qcom_qspi_transfer_one()
292 	qcom_qspi_pio_xfer(ctrl);  in qcom_qspi_transfer_one()
294 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_transfer_one()
304 	struct qcom_qspi *ctrl;  in qcom_qspi_prepare_message()  local
309 	ctrl = spi_master_get_devdata(master);  in qcom_qspi_prepare_message()
310 	spin_lock_irqsave(&ctrl->lock, flags);  in qcom_qspi_prepare_message()
312 	mstr_cfg = readl(ctrl->base + MSTR_CONFIG);  in qcom_qspi_prepare_message()
324 	writel(mstr_cfg, ctrl->base + MSTR_CONFIG);  in qcom_qspi_prepare_message()
325 	spin_unlock_irqrestore(&ctrl->lock, flags);  in qcom_qspi_prepare_message()
330 static irqreturn_t pio_read(struct qcom_qspi *ctrl)  in pio_read()  argument
341 	rd_fifo_status = readl(ctrl->base + RD_FIFO_STATUS);  in pio_read()
344 		dev_dbg(ctrl->dev, "Spurious IRQ %#x\n", rd_fifo_status);  in pio_read()
349 	wr_cnts = min(wr_cnts, ctrl->xfer.rem_bytes);  in pio_read()
355 		word_buf = ctrl->xfer.rx_buf;  in pio_read()
356 		ctrl->xfer.rem_bytes -= words_to_read * QSPI_BYTES_PER_WORD;  in pio_read()
357 		ioread32_rep(ctrl->base + RD_FIFO, word_buf, words_to_read);  in pio_read()
358 		ctrl->xfer.rx_buf = word_buf + words_to_read;  in pio_read()
362 		byte_buf = ctrl->xfer.rx_buf;  in pio_read()
363 		rd_fifo = readl(ctrl->base + RD_FIFO);  in pio_read()
364 		ctrl->xfer.rem_bytes -= bytes_to_read;  in pio_read()
367 		ctrl->xfer.rx_buf = byte_buf;  in pio_read()
373 static irqreturn_t pio_write(struct qcom_qspi *ctrl)  in pio_write()  argument
375 	const void *xfer_buf = ctrl->xfer.tx_buf;  in pio_write()
383 	wr_fifo_bytes = readl(ctrl->base + PIO_XFER_STATUS);  in pio_write()
386 	if (ctrl->xfer.rem_bytes < QSPI_BYTES_PER_WORD) {  in pio_write()
388 		wr_size = min(wr_fifo_bytes, ctrl->xfer.rem_bytes);  in pio_write()
389 		ctrl->xfer.rem_bytes -= wr_size;  in pio_write()
394 			       ctrl->base + PIO_DATAOUT_1B);  in pio_write()
395 		ctrl->xfer.tx_buf = byte_buf;  in pio_write()
402 		rem_words = ctrl->xfer.rem_bytes / QSPI_BYTES_PER_WORD;  in pio_write()
406 		ctrl->xfer.rem_bytes -= wr_size * QSPI_BYTES_PER_WORD;  in pio_write()
409 		iowrite32_rep(ctrl->base + PIO_DATAOUT_4B, word_buf, wr_size);  in pio_write()
410 		ctrl->xfer.tx_buf = word_buf + wr_size;  in pio_write()
420 	struct qcom_qspi *ctrl = dev_id;  in qcom_qspi_irq()  local
423 	spin_lock(&ctrl->lock);  in qcom_qspi_irq()
425 	int_status = readl(ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_irq()
426 	writel(int_status, ctrl->base + MSTR_INT_STATUS);  in qcom_qspi_irq()
428 	if (ctrl->xfer.dir == QSPI_WRITE) {  in qcom_qspi_irq()
430 			ret = pio_write(ctrl);  in qcom_qspi_irq()
433 			ret = pio_read(ctrl);  in qcom_qspi_irq()
438 			dev_err(ctrl->dev, "IRQ error: FIFO underrun\n");  in qcom_qspi_irq()
440 			dev_err(ctrl->dev, "IRQ error: FIFO overrun\n");  in qcom_qspi_irq()
442 			dev_err(ctrl->dev, "IRQ error: NOC response error\n");  in qcom_qspi_irq()
446 	if (!ctrl->xfer.rem_bytes) {  in qcom_qspi_irq()
447 		writel(0, ctrl->base + MSTR_INT_EN);  in qcom_qspi_irq()
448 		spi_finalize_current_transfer(dev_get_drvdata(ctrl->dev));  in qcom_qspi_irq()
451 	spin_unlock(&ctrl->lock);  in qcom_qspi_irq()
460 	struct qcom_qspi *ctrl;  in qcom_qspi_probe()  local
464 	master = devm_spi_alloc_master(dev, sizeof(*ctrl));  in qcom_qspi_probe()
470 	ctrl = spi_master_get_devdata(master);  in qcom_qspi_probe()
472 	spin_lock_init(&ctrl->lock);  in qcom_qspi_probe()
473 	ctrl->dev = dev;  in qcom_qspi_probe()
474 	ctrl->base = devm_platform_ioremap_resource(pdev, 0);  in qcom_qspi_probe()
475 	if (IS_ERR(ctrl->base))  in qcom_qspi_probe()
476 		return PTR_ERR(ctrl->base);  in qcom_qspi_probe()
478 	ctrl->clks = devm_kcalloc(dev, QSPI_NUM_CLKS,  in qcom_qspi_probe()
479 				  sizeof(*ctrl->clks), GFP_KERNEL);  in qcom_qspi_probe()
480 	if (!ctrl->clks)  in qcom_qspi_probe()
483 	ctrl->clks[QSPI_CLK_CORE].id = "core";  in qcom_qspi_probe()
484 	ctrl->clks[QSPI_CLK_IFACE].id = "iface";  in qcom_qspi_probe()
485 	ret = devm_clk_bulk_get(dev, QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_probe()
489 	ctrl->icc_path_cpu_to_qspi = devm_of_icc_get(dev, "qspi-config");  in qcom_qspi_probe()
490 	if (IS_ERR(ctrl->icc_path_cpu_to_qspi))  in qcom_qspi_probe()
491 		return dev_err_probe(dev, PTR_ERR(ctrl->icc_path_cpu_to_qspi),  in qcom_qspi_probe()
495 	ret = icc_set_bw(ctrl->icc_path_cpu_to_qspi, Bps_to_icc(1000),  in qcom_qspi_probe()
498 		dev_err(ctrl->dev, "%s: ICC BW voting failed for cpu: %d\n",  in qcom_qspi_probe()
503 	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_probe()
505 		dev_err(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",  in qcom_qspi_probe()
513 	ret = devm_request_irq(dev, ret, qcom_qspi_irq, 0, dev_name(dev), ctrl);  in qcom_qspi_probe()
570 	struct qcom_qspi *ctrl = spi_master_get_devdata(master);  in qcom_qspi_runtime_suspend()  local
575 	clk_bulk_disable_unprepare(QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_runtime_suspend()
577 	ret = icc_disable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_runtime_suspend()
579 		dev_err_ratelimited(ctrl->dev, "%s: ICC disable failed for cpu: %d\n",  in qcom_qspi_runtime_suspend()
590 	struct qcom_qspi *ctrl = spi_master_get_devdata(master);  in qcom_qspi_runtime_resume()  local
593 	ret = icc_enable(ctrl->icc_path_cpu_to_qspi);  in qcom_qspi_runtime_resume()
595 		dev_err_ratelimited(ctrl->dev, "%s: ICC enable failed for cpu: %d\n",  in qcom_qspi_runtime_resume()
600 	ret = clk_bulk_prepare_enable(QSPI_NUM_CLKS, ctrl->clks);  in qcom_qspi_runtime_resume()
604 	return dev_pm_opp_set_rate(dev, ctrl->last_speed * 4);  in qcom_qspi_runtime_resume()