201 static u32 dspi_pop_tx(struct fsl_dspi *dspi)  in dspi_pop_tx()  argument
205 	if (dspi->tx) {  in dspi_pop_tx()
206 		if (dspi->bytes_per_word == 1)  in dspi_pop_tx()
207 			txdata = *(u8 *)dspi->tx;  in dspi_pop_tx()
208 		else if (dspi->bytes_per_word == 2)  in dspi_pop_tx()
209 			txdata = *(u16 *)dspi->tx;  in dspi_pop_tx()
211 			txdata = *(u32 *)dspi->tx;  in dspi_pop_tx()
212 		dspi->tx += dspi->bytes_per_word;  in dspi_pop_tx()
214 	dspi->len -= dspi->bytes_per_word;  in dspi_pop_tx()
218 static u32 dspi_pop_tx_pushr(struct fsl_dspi *dspi)  in dspi_pop_tx_pushr()  argument
220 	u16 cmd = dspi->tx_cmd, data = dspi_pop_tx(dspi);  in dspi_pop_tx_pushr()
222 	if (spi_controller_is_slave(dspi->ctlr))  in dspi_pop_tx_pushr()
225 	if (dspi->len > 0)  in dspi_pop_tx_pushr()
230 static void dspi_push_rx(struct fsl_dspi *dspi, u32 rxdata)  in dspi_push_rx()  argument
232 	if (!dspi->rx)  in dspi_push_rx()
236 	rxdata &= (1 << dspi->bits_per_word) - 1;  in dspi_push_rx()
238 	if (dspi->bytes_per_word == 1)  in dspi_push_rx()
239 		*(u8 *)dspi->rx = rxdata;  in dspi_push_rx()
240 	else if (dspi->bytes_per_word == 2)  in dspi_push_rx()
241 		*(u16 *)dspi->rx = rxdata;  in dspi_push_rx()
243 		*(u32 *)dspi->rx = rxdata;  in dspi_push_rx()
244 	dspi->rx += dspi->bytes_per_word;  in dspi_push_rx()
249 	struct fsl_dspi *dspi = arg;  in dspi_tx_dma_callback()  local
250 	struct fsl_dspi_dma *dma = dspi->dma;  in dspi_tx_dma_callback()
257 	struct fsl_dspi *dspi = arg;  in dspi_rx_dma_callback()  local
258 	struct fsl_dspi_dma *dma = dspi->dma;  in dspi_rx_dma_callback()
261 	if (dspi->rx) {  in dspi_rx_dma_callback()
263 			dspi_push_rx(dspi, dspi->dma->rx_dma_buf[i]);  in dspi_rx_dma_callback()
269 static int dspi_next_xfer_dma_submit(struct fsl_dspi *dspi)  in dspi_next_xfer_dma_submit()  argument
271 	struct device *dev = &dspi->pdev->dev;  in dspi_next_xfer_dma_submit()
272 	struct fsl_dspi_dma *dma = dspi->dma;  in dspi_next_xfer_dma_submit()
277 		dspi->dma->tx_dma_buf[i] = dspi_pop_tx_pushr(dspi);  in dspi_next_xfer_dma_submit()
291 	dma->tx_desc->callback_param = dspi;  in dspi_next_xfer_dma_submit()
309 	dma->rx_desc->callback_param = dspi;  in dspi_next_xfer_dma_submit()
315 	reinit_completion(&dspi->dma->cmd_rx_complete);  in dspi_next_xfer_dma_submit()
316 	reinit_completion(&dspi->dma->cmd_tx_complete);  in dspi_next_xfer_dma_submit()
321 	if (spi_controller_is_slave(dspi->ctlr)) {  in dspi_next_xfer_dma_submit()
322 		wait_for_completion_interruptible(&dspi->dma->cmd_rx_complete);  in dspi_next_xfer_dma_submit()
326 	time_left = wait_for_completion_timeout(&dspi->dma->cmd_tx_complete,  in dspi_next_xfer_dma_submit()
335 	time_left = wait_for_completion_timeout(&dspi->dma->cmd_rx_complete,  in dspi_next_xfer_dma_submit()
347 static int dspi_dma_xfer(struct fsl_dspi *dspi)  in dspi_dma_xfer()  argument
349 	struct spi_message *message = dspi->cur_msg;  in dspi_dma_xfer()
350 	struct device *dev = &dspi->pdev->dev;  in dspi_dma_xfer()
351 	struct fsl_dspi_dma *dma = dspi->dma;  in dspi_dma_xfer()
356 	curr_remaining_bytes = dspi->len;  in dspi_dma_xfer()
361 			/ dspi->bytes_per_word;  in dspi_dma_xfer()
365 		ret = dspi_next_xfer_dma_submit(dspi);  in dspi_dma_xfer()
372 				dma->curr_xfer_len * dspi->bytes_per_word;  in dspi_dma_xfer()
384 static int dspi_request_dma(struct fsl_dspi *dspi, phys_addr_t phy_addr)  in dspi_request_dma()  argument
386 	struct device *dev = &dspi->pdev->dev;  in dspi_request_dma()
446 	dspi->dma = dma;  in dspi_request_dma()
464 	dspi->dma = NULL;  in dspi_request_dma()
469 static void dspi_release_dma(struct fsl_dspi *dspi)  in dspi_release_dma()  argument
471 	struct fsl_dspi_dma *dma = dspi->dma;  in dspi_release_dma()
472 	struct device *dev = &dspi->pdev->dev;  in dspi_release_dma()
561 static void fifo_write(struct fsl_dspi *dspi)  in fifo_write()  argument
563 	regmap_write(dspi->regmap, SPI_PUSHR, dspi_pop_tx_pushr(dspi));  in fifo_write()
566 static void cmd_fifo_write(struct fsl_dspi *dspi)  in cmd_fifo_write()  argument
568 	u16 cmd = dspi->tx_cmd;  in cmd_fifo_write()
570 	if (dspi->len > 0)  in cmd_fifo_write()
572 	regmap_write(dspi->regmap_pushr, PUSHR_CMD, cmd);  in cmd_fifo_write()
575 static void tx_fifo_write(struct fsl_dspi *dspi, u16 txdata)  in tx_fifo_write()  argument
577 	regmap_write(dspi->regmap_pushr, PUSHR_TX, txdata);  in tx_fifo_write()
580 static void dspi_tcfq_write(struct fsl_dspi *dspi)  in dspi_tcfq_write()  argument
583 	dspi->tx_cmd |= SPI_PUSHR_CMD_CTCNT;  in dspi_tcfq_write()
585 	if (dspi->devtype_data->xspi_mode && dspi->bits_per_word > 16) {  in dspi_tcfq_write()
589 		u32 data = dspi_pop_tx(dspi);  in dspi_tcfq_write()
591 		if (dspi->cur_chip->ctar_val & SPI_CTAR_LSBFE) {  in dspi_tcfq_write()
593 			tx_fifo_write(dspi, data & 0xFFFF);  in dspi_tcfq_write()
594 			tx_fifo_write(dspi, data >> 16);  in dspi_tcfq_write()
597 			tx_fifo_write(dspi, data >> 16);  in dspi_tcfq_write()
598 			tx_fifo_write(dspi, data & 0xFFFF);  in dspi_tcfq_write()
600 		cmd_fifo_write(dspi);  in dspi_tcfq_write()
605 		fifo_write(dspi);  in dspi_tcfq_write()
609 static u32 fifo_read(struct fsl_dspi *dspi)  in fifo_read()  argument
613 	regmap_read(dspi->regmap, SPI_POPR, &rxdata);  in fifo_read()
617 static void dspi_tcfq_read(struct fsl_dspi *dspi)  in dspi_tcfq_read()  argument
619 	dspi_push_rx(dspi, fifo_read(dspi));  in dspi_tcfq_read()
622 static void dspi_eoq_write(struct fsl_dspi *dspi)  in dspi_eoq_write()  argument
625 	u16 xfer_cmd = dspi->tx_cmd;  in dspi_eoq_write()
628 	while (dspi->len && fifo_size--) {  in dspi_eoq_write()
629 		dspi->tx_cmd = xfer_cmd;  in dspi_eoq_write()
631 		if (dspi->len == dspi->bytes_per_word || fifo_size == 0)  in dspi_eoq_write()
632 			dspi->tx_cmd |= SPI_PUSHR_CMD_EOQ;  in dspi_eoq_write()
635 			dspi->tx_cmd |= SPI_PUSHR_CMD_CTCNT;  in dspi_eoq_write()
637 		fifo_write(dspi);  in dspi_eoq_write()
641 static void dspi_eoq_read(struct fsl_dspi *dspi)  in dspi_eoq_read()  argument
646 	while ((dspi->rx < dspi->rx_end) && fifo_size--)  in dspi_eoq_read()
647 		dspi_push_rx(dspi, fifo_read(dspi));  in dspi_eoq_read()
650 static int dspi_rxtx(struct fsl_dspi *dspi)  in dspi_rxtx()  argument
652 	struct spi_message *msg = dspi->cur_msg;  in dspi_rxtx()
660 	regmap_read(dspi->regmap, SPI_TCR, &spi_tcr);  in dspi_rxtx()
663 	msg->actual_length += spi_tcnt * dspi->bytes_per_word;  in dspi_rxtx()
665 	trans_mode = dspi->devtype_data->trans_mode;  in dspi_rxtx()
667 		dspi_eoq_read(dspi);  in dspi_rxtx()
669 		dspi_tcfq_read(dspi);  in dspi_rxtx()
671 	if (!dspi->len)  in dspi_rxtx()
676 		dspi_eoq_write(dspi);  in dspi_rxtx()
678 		dspi_tcfq_write(dspi);  in dspi_rxtx()
683 static int dspi_poll(struct fsl_dspi *dspi)  in dspi_poll()  argument
689 		regmap_read(dspi->regmap, SPI_SR, &spi_sr);  in dspi_poll()
690 		regmap_write(dspi->regmap, SPI_SR, spi_sr);  in dspi_poll()
699 	return dspi_rxtx(dspi);  in dspi_poll()
704 	struct fsl_dspi *dspi = (struct fsl_dspi *)dev_id;  in dspi_interrupt()  local
707 	regmap_read(dspi->regmap, SPI_SR, &spi_sr);  in dspi_interrupt()
708 	regmap_write(dspi->regmap, SPI_SR, spi_sr);  in dspi_interrupt()
713 	if (dspi_rxtx(dspi) == 0) {  in dspi_interrupt()
714 		dspi->waitflags = 1;  in dspi_interrupt()
715 		wake_up_interruptible(&dspi->waitq);  in dspi_interrupt()
724 	struct fsl_dspi *dspi = spi_controller_get_devdata(ctlr);  in dspi_transfer_one_message()  local
733 		dspi->cur_transfer = transfer;  in dspi_transfer_one_message()
734 		dspi->cur_msg = message;  in dspi_transfer_one_message()
735 		dspi->cur_chip = spi_get_ctldata(spi);  in dspi_transfer_one_message()
737 		dspi->tx_cmd = SPI_PUSHR_CMD_CTAS(0) |  in dspi_transfer_one_message()
739 		if (list_is_last(&dspi->cur_transfer->transfer_list,  in dspi_transfer_one_message()
740 				 &dspi->cur_msg->transfers)) {  in dspi_transfer_one_message()
745 				dspi->tx_cmd |= SPI_PUSHR_CMD_CONT;  in dspi_transfer_one_message()
753 				dspi->tx_cmd |= SPI_PUSHR_CMD_CONT;  in dspi_transfer_one_message()
756 		dspi->void_write_data = dspi->cur_chip->void_write_data;  in dspi_transfer_one_message()
758 		dspi->tx = transfer->tx_buf;  in dspi_transfer_one_message()
759 		dspi->rx = transfer->rx_buf;  in dspi_transfer_one_message()
760 		dspi->rx_end = dspi->rx + transfer->len;  in dspi_transfer_one_message()
761 		dspi->len = transfer->len;  in dspi_transfer_one_message()
763 		dspi->bits_per_word = transfer->bits_per_word;  in dspi_transfer_one_message()
765 			dspi->bytes_per_word = 1;  in dspi_transfer_one_message()
767 			dspi->bytes_per_word = 2;  in dspi_transfer_one_message()
769 			dspi->bytes_per_word = 4;  in dspi_transfer_one_message()
771 		regmap_update_bits(dspi->regmap, SPI_MCR,  in dspi_transfer_one_message()
774 		regmap_write(dspi->regmap, SPI_CTAR(0),  in dspi_transfer_one_message()
775 			     dspi->cur_chip->ctar_val |  in dspi_transfer_one_message()
777 		if (dspi->devtype_data->xspi_mode)  in dspi_transfer_one_message()
778 			regmap_write(dspi->regmap, SPI_CTARE(0),  in dspi_transfer_one_message()
782 		trans_mode = dspi->devtype_data->trans_mode;  in dspi_transfer_one_message()
785 			regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_EOQFE);  in dspi_transfer_one_message()
786 			dspi_eoq_write(dspi);  in dspi_transfer_one_message()
789 			regmap_write(dspi->regmap, SPI_RSER, SPI_RSER_TCFQE);  in dspi_transfer_one_message()
790 			dspi_tcfq_write(dspi);  in dspi_transfer_one_message()
793 			regmap_write(dspi->regmap, SPI_RSER,  in dspi_transfer_one_message()
796 			status = dspi_dma_xfer(dspi);  in dspi_transfer_one_message()
799 			dev_err(&dspi->pdev->dev, "unsupported trans_mode %u\n",  in dspi_transfer_one_message()
805 		if (!dspi->irq) {  in dspi_transfer_one_message()
807 				status = dspi_poll(dspi);  in dspi_transfer_one_message()
810 			status = wait_event_interruptible(dspi->waitq,  in dspi_transfer_one_message()
811 							  dspi->waitflags);  in dspi_transfer_one_message()
812 			dspi->waitflags = 0;  in dspi_transfer_one_message()
815 			dev_err(&dspi->pdev->dev,  in dspi_transfer_one_message()
831 	struct fsl_dspi *dspi = spi_controller_get_devdata(spi->controller);  in dspi_setup()  local
847 	pdata = dev_get_platdata(&dspi->pdev->dev);  in dspi_setup()
862 	clkrate = clk_get_rate(dspi->clk);  in dspi_setup()
877 	if (!spi_controller_is_slave(dspi->ctlr)) {  in dspi_setup()
916 	struct fsl_dspi *dspi = spi_controller_get_devdata(ctlr);  in dspi_suspend()  local
919 	clk_disable_unprepare(dspi->clk);  in dspi_suspend()
929 	struct fsl_dspi *dspi = spi_controller_get_devdata(ctlr);  in dspi_resume()  local
934 	ret = clk_prepare_enable(dspi->clk);  in dspi_resume()
993 static void dspi_init(struct fsl_dspi *dspi)  in dspi_init()  argument
997 	if (dspi->devtype_data->xspi_mode)  in dspi_init()
999 	if (!spi_controller_is_slave(dspi->ctlr))  in dspi_init()
1002 	regmap_write(dspi->regmap, SPI_MCR, mcr);  in dspi_init()
1003 	regmap_write(dspi->regmap, SPI_SR, SPI_SR_CLEAR);  in dspi_init()
1004 	if (dspi->devtype_data->xspi_mode)  in dspi_init()
1005 		regmap_write(dspi->regmap, SPI_CTARE(0),  in dspi_init()
1016 	struct fsl_dspi *dspi;  in dspi_probe()  local
1024 	dspi = spi_controller_get_devdata(ctlr);  in dspi_probe()
1025 	dspi->pdev = pdev;  in dspi_probe()
1026 	dspi->ctlr = ctlr;  in dspi_probe()
1040 		dspi->devtype_data = &coldfire_data;  in dspi_probe()
1060 		dspi->devtype_data = of_device_get_match_data(&pdev->dev);  in dspi_probe()
1061 		if (!dspi->devtype_data) {  in dspi_probe()
1068 	if (dspi->devtype_data->xspi_mode)  in dspi_probe()
1080 	if (dspi->devtype_data->xspi_mode)  in dspi_probe()
1084 	dspi->regmap = devm_regmap_init_mmio(&pdev->dev, base, regmap_config);  in dspi_probe()
1085 	if (IS_ERR(dspi->regmap)) {  in dspi_probe()
1087 				PTR_ERR(dspi->regmap));  in dspi_probe()
1088 		ret = PTR_ERR(dspi->regmap);  in dspi_probe()
1092 	if (dspi->devtype_data->xspi_mode) {  in dspi_probe()
1093 		dspi->regmap_pushr = devm_regmap_init_mmio(  in dspi_probe()
1096 		if (IS_ERR(dspi->regmap_pushr)) {  in dspi_probe()
1099 				PTR_ERR(dspi->regmap_pushr));  in dspi_probe()
1100 			ret = PTR_ERR(dspi->regmap_pushr);  in dspi_probe()
1105 	dspi->clk = devm_clk_get(&pdev->dev, "dspi");  in dspi_probe()
1106 	if (IS_ERR(dspi->clk)) {  in dspi_probe()
1107 		ret = PTR_ERR(dspi->clk);  in dspi_probe()
1111 	ret = clk_prepare_enable(dspi->clk);  in dspi_probe()
1115 	dspi_init(dspi);  in dspi_probe()
1117 	dspi->irq = platform_get_irq(pdev, 0);  in dspi_probe()
1118 	if (dspi->irq <= 0) {  in dspi_probe()
1121 		dspi->irq = 0;  in dspi_probe()
1125 	ret = devm_request_irq(&pdev->dev, dspi->irq, dspi_interrupt,  in dspi_probe()
1126 			       IRQF_SHARED, pdev->name, dspi);  in dspi_probe()
1132 	init_waitqueue_head(&dspi->waitq);  in dspi_probe()
1135 	if (dspi->devtype_data->trans_mode == DSPI_DMA_MODE) {  in dspi_probe()
1136 		ret = dspi_request_dma(dspi, res->start);  in dspi_probe()
1144 		clk_get_rate(dspi->clk) / dspi->devtype_data->max_clock_factor;  in dspi_probe()
1157 	clk_disable_unprepare(dspi->clk);  in dspi_probe()
1167 	struct fsl_dspi *dspi = spi_controller_get_devdata(ctlr);  in dspi_remove()  local
1170 	dspi_release_dma(dspi);  in dspi_remove()
1171 	clk_disable_unprepare(dspi->clk);  in dspi_remove()
1172 	spi_unregister_controller(dspi->ctlr);  in dspi_remove()