1 // SPDX-License-Identifier: GPL-2.0-only
9 #include <linux/dma-mapping.h>
17 #include <linux/spi/spi-mem.h>
21 #include "spi-dw.h"
30 	u32 rx_sample_dly;	/* RX sample delay */
64 	snprintf(name, 32, "dw_spi%d", dws->master->bus_num);  in dw_spi_debugfs_init()
65 	dws->debugfs = debugfs_create_dir(name, NULL);  in dw_spi_debugfs_init()
66 	if (!dws->debugfs)  in dw_spi_debugfs_init()
67 		return -ENOMEM;  in dw_spi_debugfs_init()
69 	dws->regset.regs = dw_spi_dbgfs_regs;  in dw_spi_debugfs_init()
70 	dws->regset.nregs = ARRAY_SIZE(dw_spi_dbgfs_regs);  in dw_spi_debugfs_init()
71 	dws->regset.base = dws->regs;  in dw_spi_debugfs_init()
72 	debugfs_create_regset32("registers", 0400, dws->debugfs, &dws->regset);  in dw_spi_debugfs_init()
79 	debugfs_remove_recursive(dws->debugfs);  in dw_spi_debugfs_remove()
95 	struct dw_spi *dws = spi_controller_get_devdata(spi->controller);  in dw_spi_set_cs()
96 	bool cs_high = !!(spi->mode & SPI_CS_HIGH);  in dw_spi_set_cs()
103 	 * support active-high or active-low CS level.  in dw_spi_set_cs()
106 		dw_writel(dws, DW_SPI_SER, BIT(spi->chip_select));  in dw_spi_set_cs()
112 /* Return the max entries we can fill into tx fifo */
117 	tx_room = dws->fifo_len - dw_readl(dws, DW_SPI_TXFLR);  in dw_spi_tx_max()
120 	 * Another concern is about the tx/rx mismatch, we  in dw_spi_tx_max()
121 	 * though to use (dws->fifo_len - rxflr - txflr) as  in dw_spi_tx_max()
123 	 * data which is out of tx/rx fifo and inside the  in dw_spi_tx_max()
127 	rxtx_gap = dws->fifo_len - (dws->rx_len - dws->tx_len);  in dw_spi_tx_max()
129 	return min3((u32)dws->tx_len, tx_room, rxtx_gap);  in dw_spi_tx_max()
132 /* Return the max entries we should read out of rx fifo */
135 	return min_t(u32, dws->rx_len, dw_readl(dws, DW_SPI_RXFLR));  in dw_spi_rx_max()
143 	while (max--) {  in dw_writer()
144 		if (dws->tx) {  in dw_writer()
145 			if (dws->n_bytes == 1)  in dw_writer()
146 				txw = *(u8 *)(dws->tx);  in dw_writer()
147 			else if (dws->n_bytes == 2)  in dw_writer()
148 				txw = *(u16 *)(dws->tx);  in dw_writer()
150 				txw = *(u32 *)(dws->tx);  in dw_writer()
152 			dws->tx += dws->n_bytes;  in dw_writer()
155 		--dws->tx_len;  in dw_writer()
164 	while (max--) {  in dw_reader()
166 		if (dws->rx) {  in dw_reader()
167 			if (dws->n_bytes == 1)  in dw_reader()
168 				*(u8 *)(dws->rx) = rxw;  in dw_reader()
169 			else if (dws->n_bytes == 2)  in dw_reader()
170 				*(u16 *)(dws->rx) = rxw;  in dw_reader()
172 				*(u32 *)(dws->rx) = rxw;  in dw_reader()
174 			dws->rx += dws->n_bytes;  in dw_reader()
176 		--dws->rx_len;  in dw_reader()
191 		dev_err(&dws->master->dev, "RX FIFO overflow detected\n");  in dw_spi_check_status()
192 		ret = -EIO;  in dw_spi_check_status()
196 		dev_err(&dws->master->dev, "RX FIFO underflow detected\n");  in dw_spi_check_status()
197 		ret = -EIO;  in dw_spi_check_status()
201 		dev_err(&dws->master->dev, "TX FIFO overflow detected\n");  in dw_spi_check_status()
202 		ret = -EIO;  in dw_spi_check_status()
208 		if (dws->master->cur_msg)  in dw_spi_check_status()
209 			dws->master->cur_msg->status = ret;  in dw_spi_check_status()
221 		spi_finalize_current_transfer(dws->master);  in dw_spi_transfer_handler()
226 	 * Read data from the Rx FIFO every time we've got a chance executing  in dw_spi_transfer_handler()
228 	 * procedure. Otherwise adjust the Rx FIFO Threshold level if it's a  in dw_spi_transfer_handler()
233 	if (!dws->rx_len) {  in dw_spi_transfer_handler()
235 		spi_finalize_current_transfer(dws->master);  in dw_spi_transfer_handler()
236 	} else if (dws->rx_len <= dw_readl(dws, DW_SPI_RXFTLR)) {  in dw_spi_transfer_handler()
237 		dw_writel(dws, DW_SPI_RXFTLR, dws->rx_len - 1);  in dw_spi_transfer_handler()
241 	 * Send data out if Tx FIFO Empty IRQ is received. The IRQ will be  in dw_spi_transfer_handler()
247 		if (!dws->tx_len)  in dw_spi_transfer_handler()
263 	if (!master->cur_msg) {  in dw_spi_irq()
268 	return dws->transfer_handler(dws);  in dw_spi_irq()
284 		if (spi->mode & SPI_CPOL)  in dw_spi_prepare_cr0()
286 		if (spi->mode & SPI_CPHA)  in dw_spi_prepare_cr0()
290 		if (spi->mode & SPI_LOOP)  in dw_spi_prepare_cr0()
301 		if (spi->mode & SPI_CPOL)  in dw_spi_prepare_cr0()
303 		if (spi->mode & SPI_CPHA)  in dw_spi_prepare_cr0()
307 		if (spi->mode & SPI_LOOP)  in dw_spi_prepare_cr0()
322 	u32 cr0 = chip->cr0;  in dw_spi_update_config()
327 	cr0 |= (cfg->dfs - 1) << dws->dfs_offset;  in dw_spi_update_config()
331 		cr0 |= FIELD_PREP(DW_PSSI_CTRLR0_TMOD_MASK, cfg->tmode);  in dw_spi_update_config()
334 		cr0 |= FIELD_PREP(DW_HSSI_CTRLR0_TMOD_MASK, cfg->tmode);  in dw_spi_update_config()
338 	if (cfg->tmode == DW_SPI_CTRLR0_TMOD_EPROMREAD ||  in dw_spi_update_config()
339 	    cfg->tmode == DW_SPI_CTRLR0_TMOD_RO)  in dw_spi_update_config()
340 		dw_writel(dws, DW_SPI_CTRLR1, cfg->ndf ? cfg->ndf - 1 : 0);  in dw_spi_update_config()
343 	clk_div = (DIV_ROUND_UP(dws->max_freq, cfg->freq) + 1) & 0xfffe;  in dw_spi_update_config()
344 	speed_hz = dws->max_freq / clk_div;  in dw_spi_update_config()
346 	if (dws->current_freq != speed_hz) {  in dw_spi_update_config()
348 		dws->current_freq = speed_hz;  in dw_spi_update_config()
351 	/* Update RX sample delay if required */  in dw_spi_update_config()
352 	if (dws->cur_rx_sample_dly != chip->rx_sample_dly) {  in dw_spi_update_config()
353 		dw_writel(dws, DW_SPI_RX_SAMPLE_DLY, chip->rx_sample_dly);  in dw_spi_update_config()
354 		dws->cur_rx_sample_dly = chip->rx_sample_dly;  in dw_spi_update_config()
365 	 * Originally Tx and Rx data lengths match. Rx FIFO Threshold level  in dw_spi_irq_setup()
366 	 * will be adjusted at the final stage of the IRQ-based SPI transfer  in dw_spi_irq_setup()
369 	level = min_t(u16, dws->fifo_len / 2, dws->tx_len);  in dw_spi_irq_setup()
371 	dw_writel(dws, DW_SPI_RXFTLR, level - 1);  in dw_spi_irq_setup()
373 	dws->transfer_handler = dw_spi_transfer_handler;  in dw_spi_irq_setup()
381  * The iterative procedure of the poll-based transfer is simple: write as much
382  * as possible to the Tx FIFO, wait until the pending to receive data is ready
383  * to be read, read it from the Rx FIFO and check whether the performed
386  * Note this method the same way as the IRQ-based transfer won't work well for
388  * automatic CS assertion/de-assertion.
398 	nbits = dws->n_bytes * BITS_PER_BYTE;  in dw_spi_poll_transfer()
403 		delay.value = nbits * (dws->rx_len - dws->tx_len);  in dw_spi_poll_transfer()
411 	} while (dws->rx_len);  in dw_spi_poll_transfer()
423 		.dfs = transfer->bits_per_word,  in dw_spi_transfer_one()
424 		.freq = transfer->speed_hz,  in dw_spi_transfer_one()
428 	dws->dma_mapped = 0;  in dw_spi_transfer_one()
429 	dws->n_bytes = DIV_ROUND_UP(transfer->bits_per_word, BITS_PER_BYTE);  in dw_spi_transfer_one()
430 	dws->tx = (void *)transfer->tx_buf;  in dw_spi_transfer_one()
431 	dws->tx_len = transfer->len / dws->n_bytes;  in dw_spi_transfer_one()
432 	dws->rx = transfer->rx_buf;  in dw_spi_transfer_one()
433 	dws->rx_len = dws->tx_len;  in dw_spi_transfer_one()
442 	transfer->effective_speed_hz = dws->current_freq;  in dw_spi_transfer_one()
445 	if (master->can_dma && master->can_dma(master, spi, transfer))  in dw_spi_transfer_one()
446 		dws->dma_mapped = master->cur_msg_mapped;  in dw_spi_transfer_one()
451 	if (dws->dma_mapped) {  in dw_spi_transfer_one()
452 		ret = dws->dma_ops->dma_setup(dws, transfer);  in dw_spi_transfer_one()
459 	if (dws->dma_mapped)  in dw_spi_transfer_one()
460 		return dws->dma_ops->dma_transfer(dws, transfer);  in dw_spi_transfer_one()
461 	else if (dws->irq == IRQ_NOTCONNECTED)  in dw_spi_transfer_one()
474 	if (dws->dma_mapped)  in dw_spi_handle_err()
475 		dws->dma_ops->dma_stop(dws);  in dw_spi_handle_err()
482 	if (op->data.dir == SPI_MEM_DATA_IN)  in dw_spi_adjust_mem_op_size()
483 		op->data.nbytes = clamp_val(op->data.nbytes, 0, DW_SPI_NDF_MASK + 1);  in dw_spi_adjust_mem_op_size()
491 	if (op->data.buswidth > 1 || op->addr.buswidth > 1 ||  in dw_spi_supports_mem_op()
492 	    op->dummy.buswidth > 1 || op->cmd.buswidth > 1)  in dw_spi_supports_mem_op()
505 	 * either use the pre-allocated buffer or create a temporary one.  in dw_spi_init_mem_buf()
507 	len = op->cmd.nbytes + op->addr.nbytes + op->dummy.nbytes;  in dw_spi_init_mem_buf()
508 	if (op->data.dir == SPI_MEM_DATA_OUT)  in dw_spi_init_mem_buf()
509 		len += op->data.nbytes;  in dw_spi_init_mem_buf()
512 		out = dws->buf;  in dw_spi_init_mem_buf()
516 			return -ENOMEM;  in dw_spi_init_mem_buf()
524 	for (i = 0; i < op->cmd.nbytes; ++i)  in dw_spi_init_mem_buf()
525 		out[i] = DW_SPI_GET_BYTE(op->cmd.opcode, op->cmd.nbytes - i - 1);  in dw_spi_init_mem_buf()
526 	for (j = 0; j < op->addr.nbytes; ++i, ++j)  in dw_spi_init_mem_buf()
527 		out[i] = DW_SPI_GET_BYTE(op->addr.val, op->addr.nbytes - j - 1);  in dw_spi_init_mem_buf()
528 	for (j = 0; j < op->dummy.nbytes; ++i, ++j)  in dw_spi_init_mem_buf()
531 	if (op->data.dir == SPI_MEM_DATA_OUT)  in dw_spi_init_mem_buf()
532 		memcpy(&out[i], op->data.buf.out, op->data.nbytes);  in dw_spi_init_mem_buf()
534 	dws->n_bytes = 1;  in dw_spi_init_mem_buf()
535 	dws->tx = out;  in dw_spi_init_mem_buf()
536 	dws->tx_len = len;  in dw_spi_init_mem_buf()
537 	if (op->data.dir == SPI_MEM_DATA_IN) {  in dw_spi_init_mem_buf()
538 		dws->rx = op->data.buf.in;  in dw_spi_init_mem_buf()
539 		dws->rx_len = op->data.nbytes;  in dw_spi_init_mem_buf()
541 		dws->rx = NULL;  in dw_spi_init_mem_buf()
542 		dws->rx_len = 0;  in dw_spi_init_mem_buf()
550 	if (dws->tx != dws->buf)  in dw_spi_free_mem_buf()
551 		kfree(dws->tx);  in dw_spi_free_mem_buf()
561 	 * At initial stage we just pre-fill the Tx FIFO in with no rush,  in dw_spi_write_then_read()
565 	len = min(dws->fifo_len, dws->tx_len);  in dw_spi_write_then_read()
566 	buf = dws->tx;  in dw_spi_write_then_read()
567 	while (len--)  in dw_spi_write_then_read()
573 	 * otherwise the CS de-assertion will happen whereupon the memory  in dw_spi_write_then_read()
574 	 * operation will be pre-terminated.  in dw_spi_write_then_read()
576 	len = dws->tx_len - ((void *)buf - dws->tx);  in dw_spi_write_then_read()
579 		entries = readl_relaxed(dws->regs + DW_SPI_TXFLR);  in dw_spi_write_then_read()
581 			dev_err(&dws->master->dev, "CS de-assertion on Tx\n");  in dw_spi_write_then_read()
582 			return -EIO;  in dw_spi_write_then_read()
584 		room = min(dws->fifo_len - entries, len);  in dw_spi_write_then_read()
585 		for (; room; --room, --len)  in dw_spi_write_then_read()
590 	 * Data fetching will start automatically if the EEPROM-read mode is  in dw_spi_write_then_read()
592 	 * prevent the Rx FIFO overflow causing the inbound data loss.  in dw_spi_write_then_read()
594 	len = dws->rx_len;  in dw_spi_write_then_read()
595 	buf = dws->rx;  in dw_spi_write_then_read()
597 		entries = readl_relaxed(dws->regs + DW_SPI_RXFLR);  in dw_spi_write_then_read()
599 			sts = readl_relaxed(dws->regs + DW_SPI_RISR);  in dw_spi_write_then_read()
601 				dev_err(&dws->master->dev, "FIFO overflow on Rx\n");  in dw_spi_write_then_read()
602 				return -EIO;  in dw_spi_write_then_read()
607 		for (; entries; --entries, --len)  in dw_spi_write_then_read()
627 	ns = NSEC_PER_SEC / dws->current_freq * nents;  in dw_spi_wait_mem_op_done()
628 	ns *= dws->n_bytes * BITS_PER_BYTE;  in dw_spi_wait_mem_op_done()
638 	while (dw_spi_ctlr_busy(dws) && retry--)  in dw_spi_wait_mem_op_done()
642 		dev_err(&dws->master->dev, "Mem op hanged up\n");  in dw_spi_wait_mem_op_done()
643 		return -EIO;  in dw_spi_wait_mem_op_done()
658  * devices, which are selected by the native chip-select lane. It's
659  * specifically developed to workaround the problem with automatic chip-select
660  * lane toggle when there is no data in the Tx FIFO buffer. Luckily the current
661  * SPI-mem core calls exec_op() callback only if the GPIO-based CS is
666 	struct dw_spi *dws = spi_controller_get_devdata(mem->spi->controller);  in dw_spi_exec_mem_op()
680 	 * DW SPI EEPROM-read mode is required only for the SPI memory Data-IN  in dw_spi_exec_mem_op()
681 	 * operation. Transmit-only mode is suitable for the rest of them.  in dw_spi_exec_mem_op()
684 	cfg.freq = clamp(mem->spi->max_speed_hz, 0U, dws->max_mem_freq);  in dw_spi_exec_mem_op()
685 	if (op->data.dir == SPI_MEM_DATA_IN) {  in dw_spi_exec_mem_op()
687 		cfg.ndf = op->data.nbytes;  in dw_spi_exec_mem_op()
694 	dw_spi_update_config(dws, mem->spi, &cfg);  in dw_spi_exec_mem_op()
702 	 * (without any vendor-specific modifications) it doesn't provide a  in dw_spi_exec_mem_op()
703 	 * direct way to set and clear the native chip-select signal. Instead  in dw_spi_exec_mem_op()
704 	 * the controller asserts the CS lane if Tx FIFO isn't empty and a  in dw_spi_exec_mem_op()
705 	 * transmission is going on, and automatically de-asserts it back to  in dw_spi_exec_mem_op()
706 	 * the high level if the Tx FIFO doesn't have anything to be pushed  in dw_spi_exec_mem_op()
707 	 * out. Due to that a multi-tasking or heavy IRQs activity might be  in dw_spi_exec_mem_op()
708 	 * fatal, since the transfer procedure preemption may cause the Tx FIFO  in dw_spi_exec_mem_op()
709 	 * getting empty and sudden CS de-assertion, which in the middle of the  in dw_spi_exec_mem_op()
711 	 * EEPROM-read or Read-only DW SPI transfer modes imply the incoming  in dw_spi_exec_mem_op()
712 	 * data being automatically pulled in into the Rx FIFO. So if the  in dw_spi_exec_mem_op()
713 	 * driver software is late in fetching the data from the FIFO before  in dw_spi_exec_mem_op()
715 	 * sure the executed memory operations are CS-atomic and to prevent the  in dw_spi_exec_mem_op()
716 	 * Rx FIFO overflow we have to disable the local interrupts so to block  in dw_spi_exec_mem_op()
720 	 * the problems described above. The CS de-assertion and Rx FIFO  in dw_spi_exec_mem_op()
722 	 * CPU not working fast enough, so the write-then-read algo implemented  in dw_spi_exec_mem_op()
726 	 * dws->max_mem_freq parameter.  in dw_spi_exec_mem_op()
731 	ret = dw_spi_write_then_read(dws, mem->spi);  in dw_spi_exec_mem_op()
738 	 * status only if there hasn't been any run-time error detected. In the  in dw_spi_exec_mem_op()
749 	dw_spi_stop_mem_op(dws, mem->spi);  in dw_spi_exec_mem_op()
762  * has fixed the automatic CS assertion/de-assertion peculiarity, then it will
763  * be safer to use the normal SPI-messages-based transfers implementation.
767 	if (!dws->mem_ops.exec_op && !(dws->caps & DW_SPI_CAP_CS_OVERRIDE) &&  in dw_spi_init_mem_ops()
768 	    !dws->set_cs) {  in dw_spi_init_mem_ops()
769 		dws->mem_ops.adjust_op_size = dw_spi_adjust_mem_op_size;  in dw_spi_init_mem_ops()
770 		dws->mem_ops.supports_op = dw_spi_supports_mem_op;  in dw_spi_init_mem_ops()
771 		dws->mem_ops.exec_op = dw_spi_exec_mem_op;  in dw_spi_init_mem_ops()
772 		if (!dws->max_mem_freq)  in dw_spi_init_mem_ops()
773 			dws->max_mem_freq = dws->max_freq;  in dw_spi_init_mem_ops()
780 	struct dw_spi *dws = spi_controller_get_devdata(spi->controller);  in dw_spi_setup()
786 		struct dw_spi *dws = spi_controller_get_devdata(spi->controller);  in dw_spi_setup()
791 			return -ENOMEM;  in dw_spi_setup()
793 		/* Get specific / default rx-sample-delay */  in dw_spi_setup()
794 		if (device_property_read_u32(&spi->dev,  in dw_spi_setup()
795 					     "rx-sample-delay-ns",  in dw_spi_setup()
798 			rx_sample_dly_ns = dws->def_rx_sample_dly_ns;  in dw_spi_setup()
799 		chip->rx_sample_dly = DIV_ROUND_CLOSEST(rx_sample_dly_ns,  in dw_spi_setup()
801 							dws->max_freq);  in dw_spi_setup()
809 	chip->cr0 = dw_spi_prepare_cr0(dws, spi);  in dw_spi_setup()
822 /* Restart the controller, disable all interrupts, clean rx fifo */
829 	 * by the platform. CoreKit version ID is encoded as a 3-chars ASCII  in dw_spi_hw_init()
830 	 * code enclosed with '*' (typical for the most of Synopsys IP-cores).  in dw_spi_hw_init()
832 	if (!dws->ver) {  in dw_spi_hw_init()
833 		dws->ver = dw_readl(dws, DW_SPI_VERSION);  in dw_spi_hw_init()
837 			DW_SPI_GET_BYTE(dws->ver, 3), DW_SPI_GET_BYTE(dws->ver, 2),  in dw_spi_hw_init()
838 			DW_SPI_GET_BYTE(dws->ver, 1));  in dw_spi_hw_init()
842 	 * Try to detect the FIFO depth if not set by interface driver,  in dw_spi_hw_init()
843 	 * the depth could be from 2 to 256 from HW spec  in dw_spi_hw_init()
845 	if (!dws->fifo_len) {  in dw_spi_hw_init()
846 		u32 fifo;  in dw_spi_hw_init()  local
848 		for (fifo = 1; fifo < 256; fifo++) {  in dw_spi_hw_init()
849 			dw_writel(dws, DW_SPI_TXFTLR, fifo);  in dw_spi_hw_init()
850 			if (fifo != dw_readl(dws, DW_SPI_TXFTLR))  in dw_spi_hw_init()
855 		dws->fifo_len = (fifo == 1) ? 0 : fifo;  in dw_spi_hw_init()
856 		dev_dbg(dev, "Detected FIFO size: %u bytes\n", dws->fifo_len);  in dw_spi_hw_init()
874 			dws->caps |= DW_SPI_CAP_DFS32;  in dw_spi_hw_init()
875 			dws->dfs_offset = __bf_shf(DW_PSSI_CTRLR0_DFS32_MASK);  in dw_spi_hw_init()
876 			dev_dbg(dev, "Detected 32-bits max data frame size\n");  in dw_spi_hw_init()
879 		dws->caps |= DW_SPI_CAP_DFS32;  in dw_spi_hw_init()
883 	if (dws->caps & DW_SPI_CAP_CS_OVERRIDE)  in dw_spi_hw_init()
893 		return -EINVAL;  in dw_spi_add_host()
897 		return -ENOMEM;  in dw_spi_add_host()
899 	device_set_node(&master->dev, dev_fwnode(dev));  in dw_spi_add_host()
901 	dws->master = master;  in dw_spi_add_host()
902 	dws->dma_addr = (dma_addr_t)(dws->paddr + DW_SPI_DR);  in dw_spi_add_host()
909 	ret = request_irq(dws->irq, dw_spi_irq, IRQF_SHARED, dev_name(dev),  in dw_spi_add_host()
911 	if (ret < 0 && ret != -ENOTCONN) {  in dw_spi_add_host()
918 	master->use_gpio_descriptors = true;  in dw_spi_add_host()
919 	master->mode_bits = SPI_CPOL | SPI_CPHA | SPI_LOOP;  in dw_spi_add_host()
920 	if (dws->caps & DW_SPI_CAP_DFS32)  in dw_spi_add_host()
921 		master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 32);  in dw_spi_add_host()
923 		master->bits_per_word_mask = SPI_BPW_RANGE_MASK(4, 16);  in dw_spi_add_host()
924 	master->bus_num = dws->bus_num;  in dw_spi_add_host()
925 	master->num_chipselect = dws->num_cs;  in dw_spi_add_host()
926 	master->setup = dw_spi_setup;  in dw_spi_add_host()
927 	master->cleanup = dw_spi_cleanup;  in dw_spi_add_host()
928 	if (dws->set_cs)  in dw_spi_add_host()
929 		master->set_cs = dws->set_cs;  in dw_spi_add_host()
931 		master->set_cs = dw_spi_set_cs;  in dw_spi_add_host()
932 	master->transfer_one = dw_spi_transfer_one;  in dw_spi_add_host()
933 	master->handle_err = dw_spi_handle_err;  in dw_spi_add_host()
934 	if (dws->mem_ops.exec_op)  in dw_spi_add_host()
935 		master->mem_ops = &dws->mem_ops;  in dw_spi_add_host()
936 	master->max_speed_hz = dws->max_freq;  in dw_spi_add_host()
937 	master->flags = SPI_MASTER_GPIO_SS;  in dw_spi_add_host()
938 	master->auto_runtime_pm = true;  in dw_spi_add_host()
940 	/* Get default rx sample delay */  in dw_spi_add_host()
941 	device_property_read_u32(dev, "rx-sample-delay-ns",  in dw_spi_add_host()
942 				 &dws->def_rx_sample_dly_ns);  in dw_spi_add_host()
944 	if (dws->dma_ops && dws->dma_ops->dma_init) {  in dw_spi_add_host()
945 		ret = dws->dma_ops->dma_init(dev, dws);  in dw_spi_add_host()
946 		if (ret == -EPROBE_DEFER) {  in dw_spi_add_host()
951 			master->can_dma = dws->dma_ops->can_dma;  in dw_spi_add_host()
952 			master->flags |= SPI_CONTROLLER_MUST_TX;  in dw_spi_add_host()
966 	if (dws->dma_ops && dws->dma_ops->dma_exit)  in dw_spi_add_host()
967 		dws->dma_ops->dma_exit(dws);  in dw_spi_add_host()
970 	free_irq(dws->irq, master);  in dw_spi_add_host()
981 	spi_unregister_controller(dws->master);  in dw_spi_remove_host()
983 	if (dws->dma_ops && dws->dma_ops->dma_exit)  in dw_spi_remove_host()
984 		dws->dma_ops->dma_exit(dws);  in dw_spi_remove_host()
988 	free_irq(dws->irq, dws->master);  in dw_spi_remove_host()
996 	ret = spi_controller_suspend(dws->master);  in dw_spi_suspend_host()
1007 	dw_spi_hw_init(&dws->master->dev, dws);  in dw_spi_resume_host()
1008 	return spi_controller_resume(dws->master);  in dw_spi_resume_host()