1 // SPDX-License-Identifier: GPL-2.0-or-later
10  * spi-ath79.c, Copyright (C) 2009-2011 Gabor Juhos <juhosg@openwrt.org>
11  * spi-atmel.c, Copyright (C) 2006 Atmel Corporation
18 #include <linux/dma-mapping.h>
75 #define DRV_NAME	"spi-bcm2835"
84  * struct bcm2835_spi - BCM2835 SPI controller
87  * @irq: interrupt, signals TX FIFO empty or RX FIFO ¾ full
96  * @rx_prologue: bytes received without DMA if first RX sglist entry's
99  * @prepare_cs: precalculated CS register value for ->prepare_message()
100  *	(uses slave-specific clock polarity and phase settings)
101  * @debugfs_dir: the debugfs directory - neede to remove debugfs when
113  * @rx_dma_active: whether a RX DMA descriptor is in progress
115  * @fill_tx_desc: preallocated TX DMA descriptor used for RX-only transfers
116  *	(cyclically copies from zero page to TX FIFO)
118  * @clear_rx_desc: preallocated RX DMA descriptor used for TX-only transfers
119  *	(cyclically clears RX FIFO by writing @clear_rx_cs to CS register)
121  * @clear_rx_cs: precalculated CS register value to clear RX FIFO
122  *	(uses slave-specific clock polarity and phase settings)
163 	snprintf(name, sizeof(name), "spi-bcm2835-%s", dname);  in bcm2835_debugfs_create()
167 	bs->debugfs_dir = dir;  in bcm2835_debugfs_create()
171 			   &bs->count_transfer_polling);  in bcm2835_debugfs_create()
173 			   &bs->count_transfer_irq);  in bcm2835_debugfs_create()
175 			   &bs->count_transfer_irq_after_polling);  in bcm2835_debugfs_create()
177 			   &bs->count_transfer_dma);  in bcm2835_debugfs_create()
182 	debugfs_remove_recursive(bs->debugfs_dir);  in bcm2835_debugfs_remove()
183 	bs->debugfs_dir = NULL;  in bcm2835_debugfs_remove()
198 	return readl(bs->regs + reg);  in bcm2835_rd()
203 	writel(val, bs->regs + reg);  in bcm2835_wr()
210 	while ((bs->rx_len) &&  in bcm2835_rd_fifo()
213 		if (bs->rx_buf)  in bcm2835_rd_fifo()
214 			*bs->rx_buf++ = byte;  in bcm2835_rd_fifo()
215 		bs->rx_len--;  in bcm2835_rd_fifo()
223 	while ((bs->tx_len) &&  in bcm2835_wr_fifo()
225 		byte = bs->tx_buf ? *bs->tx_buf++ : 0;  in bcm2835_wr_fifo()
227 		bs->tx_len--;  in bcm2835_wr_fifo()
232  * bcm2835_rd_fifo_count() - blindly read exactly @count bytes from RX FIFO
234  * @count: bytes to read from RX FIFO
236  * The caller must ensure that @bs->rx_len is greater than or equal to @count,
237  * that the RX FIFO contains at least @count bytes and that the DMA Enable flag
238  * in the CS register is set (such that a read from the FIFO register receives
239  * 32-bit instead of just 8-bit).  Moreover @bs->rx_buf must not be %NULL.
246 	bs->rx_len -= count;  in bcm2835_rd_fifo_count()
251 		memcpy(bs->rx_buf, &val, len);  in bcm2835_rd_fifo_count()
252 		bs->rx_buf += len;  in bcm2835_rd_fifo_count()
253 		count -= 4;  in bcm2835_rd_fifo_count()
258  * bcm2835_wr_fifo_count() - blindly write exactly @count bytes to TX FIFO
260  * @count: bytes to write to TX FIFO
262  * The caller must ensure that @bs->tx_len is greater than or equal to @count,
263  * that the TX FIFO can accommodate @count bytes and that the DMA Enable flag
264  * in the CS register is set (such that a write to the FIFO register transmits
265  * 32-bit instead of just 8-bit).
272 	bs->tx_len -= count;  in bcm2835_wr_fifo_count()
275 		if (bs->tx_buf) {  in bcm2835_wr_fifo_count()
277 			memcpy(&val, bs->tx_buf, len);  in bcm2835_wr_fifo_count()
278 			bs->tx_buf += len;  in bcm2835_wr_fifo_count()
283 		count -= 4;  in bcm2835_wr_fifo_count()
288  * bcm2835_wait_tx_fifo_empty() - busy-wait for TX FIFO to empty
291  * The caller must ensure that the RX FIFO can accommodate as many bytes
292  * as have been written to the TX FIFO:  Transmission is halted once the
293  * RX FIFO is full, causing this function to spin forever.
302  * bcm2835_rd_fifo_blind() - blindly read up to @count bytes from RX FIFO
304  * @count: bytes available for reading in RX FIFO
310 	count = min(count, bs->rx_len);  in bcm2835_rd_fifo_blind()
311 	bs->rx_len -= count;  in bcm2835_rd_fifo_blind()
315 		if (bs->rx_buf)  in bcm2835_rd_fifo_blind()
316 			*bs->rx_buf++ = val;  in bcm2835_rd_fifo_blind()
317 	} while (--count);  in bcm2835_rd_fifo_blind()
321  * bcm2835_wr_fifo_blind() - blindly write up to @count bytes to TX FIFO
323  * @count: bytes available for writing in TX FIFO
329 	count = min(count, bs->tx_len);  in bcm2835_wr_fifo_blind()
330 	bs->tx_len -= count;  in bcm2835_wr_fifo_blind()
333 		val = bs->tx_buf ? *bs->tx_buf++ : 0;  in bcm2835_wr_fifo_blind()
335 	} while (--count);  in bcm2835_wr_fifo_blind()
354 	/* and reset RX/TX FIFOS */  in bcm2835_spi_reset_hw()
369 	 * An interrupt is signaled either if DONE is set (TX FIFO empty)  in bcm2835_spi_interrupt()
370 	 * or if RXR is set (RX FIFO >= ¾ full).  in bcm2835_spi_interrupt()
377 	if (bs->tx_len && cs & BCM2835_SPI_CS_DONE)  in bcm2835_spi_interrupt()
380 	/* Read as many bytes as possible from FIFO */  in bcm2835_spi_interrupt()
382 	/* Write as many bytes as possible to FIFO */  in bcm2835_spi_interrupt()
385 	if (!bs->rx_len) {  in bcm2835_spi_interrupt()
386 		/* Transfer complete - reset SPI HW */  in bcm2835_spi_interrupt()
389 		complete(&bs->ctlr->xfer_completion);  in bcm2835_spi_interrupt()
403 	bs->count_transfer_irq++;  in bcm2835_spi_transfer_one_irq()
407 	 * Otherwise the empty TX FIFO would immediately trigger an interrupt.  in bcm2835_spi_transfer_one_irq()
411 	/* fill TX FIFO as much as possible */  in bcm2835_spi_transfer_one_irq()
425  * bcm2835_spi_transfer_prologue() - transfer first few bytes without DMA
431  * A limitation in DMA mode is that the FIFO must be accessed in 4 byte chunks.
433  * SPI controller deduces its intended size from the DLEN register.
435  * If a TX or RX sglist contains multiple entries, one per page, and the first
447  * rounded up a to a multiple of 4 bytes by transmitting surplus bytes, an RX
451  * E.g. if the first TX sglist entry's length is 23 and the first RX's is 42,
452  * write 3 bytes to the TX FIFO but read only 2 bytes from the RX FIFO.
453  * The residue of 1 byte in the RX FIFO is picked up by DMA.  Together with
454  * the rest of the first RX sglist entry it makes up a multiple of 4 bytes.
456  * Should the RX prologue be larger, say, 3 vis-à-vis a TX prologue of 1,
457  * write 1 + 4 = 5 bytes to the TX FIFO and read 3 bytes from the RX FIFO.
464  * The FIFO is normally accessed with 8-bit width by the CPU and 32-bit width
466  * the width but also garbles the FIFO's contents.  The prologue must therefore
467  * be transmitted in 32-bit width to ensure that the following DMA transfer can
468  * pick up the residue in the RX FIFO in ungarbled form.
477 	bs->tfr		 = tfr;  in bcm2835_spi_transfer_prologue()
478 	bs->tx_prologue  = 0;  in bcm2835_spi_transfer_prologue()
479 	bs->rx_prologue  = 0;  in bcm2835_spi_transfer_prologue()
480 	bs->tx_spillover = false;  in bcm2835_spi_transfer_prologue()
482 	if (bs->tx_buf && !sg_is_last(&tfr->tx_sg.sgl[0]))  in bcm2835_spi_transfer_prologue()
483 		bs->tx_prologue = sg_dma_len(&tfr->tx_sg.sgl[0]) & 3;  in bcm2835_spi_transfer_prologue()
485 	if (bs->rx_buf && !sg_is_last(&tfr->rx_sg.sgl[0])) {  in bcm2835_spi_transfer_prologue()
486 		bs->rx_prologue = sg_dma_len(&tfr->rx_sg.sgl[0]) & 3;  in bcm2835_spi_transfer_prologue()
488 		if (bs->rx_prologue > bs->tx_prologue) {  in bcm2835_spi_transfer_prologue()
489 			if (!bs->tx_buf || sg_is_last(&tfr->tx_sg.sgl[0])) {  in bcm2835_spi_transfer_prologue()
490 				bs->tx_prologue  = bs->rx_prologue;  in bcm2835_spi_transfer_prologue()
492 				bs->tx_prologue += 4;  in bcm2835_spi_transfer_prologue()
493 				bs->tx_spillover =  in bcm2835_spi_transfer_prologue()
494 					!(sg_dma_len(&tfr->tx_sg.sgl[0]) & ~3);  in bcm2835_spi_transfer_prologue()
500 	if (!bs->tx_prologue)  in bcm2835_spi_transfer_prologue()
503 	/* Write and read RX prologue.  Adjust first entry in RX sglist. */  in bcm2835_spi_transfer_prologue()
504 	if (bs->rx_prologue) {  in bcm2835_spi_transfer_prologue()
505 		bcm2835_wr(bs, BCM2835_SPI_DLEN, bs->rx_prologue);  in bcm2835_spi_transfer_prologue()
508 		bcm2835_wr_fifo_count(bs, bs->rx_prologue);  in bcm2835_spi_transfer_prologue()
510 		bcm2835_rd_fifo_count(bs, bs->rx_prologue);  in bcm2835_spi_transfer_prologue()
515 		dma_sync_single_for_device(ctlr->dma_rx->device->dev,  in bcm2835_spi_transfer_prologue()
516 					   sg_dma_address(&tfr->rx_sg.sgl[0]),  in bcm2835_spi_transfer_prologue()
517 					   bs->rx_prologue, DMA_FROM_DEVICE);  in bcm2835_spi_transfer_prologue()
519 		sg_dma_address(&tfr->rx_sg.sgl[0]) += bs->rx_prologue;  in bcm2835_spi_transfer_prologue()
520 		sg_dma_len(&tfr->rx_sg.sgl[0])     -= bs->rx_prologue;  in bcm2835_spi_transfer_prologue()
523 	if (!bs->tx_buf)  in bcm2835_spi_transfer_prologue()
530 	tx_remaining = bs->tx_prologue - bs->rx_prologue;  in bcm2835_spi_transfer_prologue()
541 	if (likely(!bs->tx_spillover)) {  in bcm2835_spi_transfer_prologue()
542 		sg_dma_address(&tfr->tx_sg.sgl[0]) += bs->tx_prologue;  in bcm2835_spi_transfer_prologue()
543 		sg_dma_len(&tfr->tx_sg.sgl[0])     -= bs->tx_prologue;  in bcm2835_spi_transfer_prologue()
545 		sg_dma_len(&tfr->tx_sg.sgl[0])      = 0;  in bcm2835_spi_transfer_prologue()
546 		sg_dma_address(&tfr->tx_sg.sgl[1]) += 4;  in bcm2835_spi_transfer_prologue()
547 		sg_dma_len(&tfr->tx_sg.sgl[1])     -= 4;  in bcm2835_spi_transfer_prologue()
552  * bcm2835_spi_undo_prologue() - reconstruct original sglist state
561 	struct spi_transfer *tfr = bs->tfr;  in bcm2835_spi_undo_prologue()
563 	if (!bs->tx_prologue)  in bcm2835_spi_undo_prologue()
566 	if (bs->rx_prologue) {  in bcm2835_spi_undo_prologue()
567 		sg_dma_address(&tfr->rx_sg.sgl[0]) -= bs->rx_prologue;  in bcm2835_spi_undo_prologue()
568 		sg_dma_len(&tfr->rx_sg.sgl[0])     += bs->rx_prologue;  in bcm2835_spi_undo_prologue()
571 	if (!bs->tx_buf)  in bcm2835_spi_undo_prologue()
574 	if (likely(!bs->tx_spillover)) {  in bcm2835_spi_undo_prologue()
575 		sg_dma_address(&tfr->tx_sg.sgl[0]) -= bs->tx_prologue;  in bcm2835_spi_undo_prologue()
576 		sg_dma_len(&tfr->tx_sg.sgl[0])     += bs->tx_prologue;  in bcm2835_spi_undo_prologue()
578 		sg_dma_len(&tfr->tx_sg.sgl[0])      = bs->tx_prologue - 4;  in bcm2835_spi_undo_prologue()
579 		sg_dma_address(&tfr->tx_sg.sgl[1]) -= 4;  in bcm2835_spi_undo_prologue()
580 		sg_dma_len(&tfr->tx_sg.sgl[1])     += 4;  in bcm2835_spi_undo_prologue()
583 	bs->tx_prologue = 0;  in bcm2835_spi_undo_prologue()
587  * bcm2835_spi_dma_rx_done() - callback for DMA RX channel
590  * Used for bidirectional and RX-only transfers.
597 	/* terminate tx-dma as we do not have an irq for it  in bcm2835_spi_dma_rx_done()
598 	 * because when the rx dma will terminate and this callback  in bcm2835_spi_dma_rx_done()
599 	 * is called the tx-dma must have finished - can't get to this  in bcm2835_spi_dma_rx_done()
602 	dmaengine_terminate_async(ctlr->dma_tx);  in bcm2835_spi_dma_rx_done()
603 	bs->tx_dma_active = false;  in bcm2835_spi_dma_rx_done()
604 	bs->rx_dma_active = false;  in bcm2835_spi_dma_rx_done()
607 	/* reset fifo and HW */  in bcm2835_spi_dma_rx_done()
611 	complete(&ctlr->xfer_completion);  in bcm2835_spi_dma_rx_done()
615  * bcm2835_spi_dma_tx_done() - callback for DMA TX channel
618  * Used for TX-only transfers.
625 	/* busy-wait for TX FIFO to empty */  in bcm2835_spi_dma_tx_done()
628 			   bs->clear_rx_cs[bs->chip_select]);  in bcm2835_spi_dma_tx_done()
630 	bs->tx_dma_active = false;  in bcm2835_spi_dma_tx_done()
634 	 * In case of a very short transfer, RX DMA may not have been  in bcm2835_spi_dma_tx_done()
638 	if (cmpxchg(&bs->rx_dma_active, true, false))  in bcm2835_spi_dma_tx_done()
639 		dmaengine_terminate_async(ctlr->dma_rx);  in bcm2835_spi_dma_tx_done()
643 	complete(&ctlr->xfer_completion);  in bcm2835_spi_dma_tx_done()
647  * bcm2835_spi_prepare_sg() - prepare and submit DMA descriptor for sglist
652  * @is_tx: whether to submit DMA descriptor for TX or RX sglist
654  * Prepare and submit a DMA descriptor for the TX or RX sglist of @tfr.
674 		chan  = ctlr->dma_tx;  in bcm2835_spi_prepare_sg()
675 		nents = tfr->tx_sg.nents;  in bcm2835_spi_prepare_sg()
676 		sgl   = tfr->tx_sg.sgl;  in bcm2835_spi_prepare_sg()
677 		flags = tfr->rx_buf ? 0 : DMA_PREP_INTERRUPT;  in bcm2835_spi_prepare_sg()
680 		chan  = ctlr->dma_rx;  in bcm2835_spi_prepare_sg()
681 		nents = tfr->rx_sg.nents;  in bcm2835_spi_prepare_sg()
682 		sgl   = tfr->rx_sg.sgl;  in bcm2835_spi_prepare_sg()
688 		return -EINVAL;  in bcm2835_spi_prepare_sg()
691 	 * Completion is signaled by the RX channel for bidirectional and  in bcm2835_spi_prepare_sg()
692 	 * RX-only transfers; else by the TX channel for TX-only transfers.  in bcm2835_spi_prepare_sg()
695 		desc->callback = bcm2835_spi_dma_rx_done;  in bcm2835_spi_prepare_sg()
696 		desc->callback_param = ctlr;  in bcm2835_spi_prepare_sg()
697 	} else if (!tfr->rx_buf) {  in bcm2835_spi_prepare_sg()
698 		desc->callback = bcm2835_spi_dma_tx_done;  in bcm2835_spi_prepare_sg()
699 		desc->callback_param = ctlr;  in bcm2835_spi_prepare_sg()
700 		bs->chip_select = spi->chip_select;  in bcm2835_spi_prepare_sg()
703 	/* submit it to DMA-engine */  in bcm2835_spi_prepare_sg()
710  * bcm2835_spi_transfer_one_dma() - perform SPI transfer using DMA engine
716  * For *bidirectional* transfers (both tx_buf and rx_buf are non-%NULL), set up
717  * the TX and RX DMA channel to copy between memory and FIFO register.
719  * For *TX-only* transfers (rx_buf is %NULL), copying the RX FIFO's contents to
720  * memory is pointless.  However not reading the RX FIFO isn't an option either
722  * clear the RX FIFO by setting the CLEAR_RX bit in the CS register.
727  * when performing a TX-only transfer is to submit this descriptor to the RX
732  * Clearing the RX FIFO is paced by the DREQ signal.  The signal is asserted
733  * when the RX FIFO becomes half full, i.e. 32 bytes.  (Tuneable with the DC
734  * register.)  Reading 32 bytes from the RX FIFO would normally require 8 bus
735  * accesses, whereas clearing it requires only 1 bus access.  So an 8-fold
738  * For *RX-only* transfers (tx_buf is %NULL), fill the TX FIFO by cyclically
740  * in bcm2835_dma_init().  It must be terminated once the RX DMA channel is
746  * feature is not available on so-called "lite" channels, but normally TX DMA
747  * is backed by a full-featured channel.
749  * Zero-filling the TX FIFO is paced by the DREQ signal.  Unfortunately the
752  * has finished, the DMA engine zero-fills the TX FIFO until it is half full.
754  * performed at the end of an RX-only transfer.
766 	bs->count_transfer_dma++;  in bcm2835_spi_transfer_one_dma()
769 	 * Transfer first few bytes without DMA if length of first TX or RX  in bcm2835_spi_transfer_one_dma()
774 	/* setup tx-DMA */  in bcm2835_spi_transfer_one_dma()
775 	if (bs->tx_buf) {  in bcm2835_spi_transfer_one_dma()
778 		cookie = dmaengine_submit(bs->fill_tx_desc);  in bcm2835_spi_transfer_one_dma()
785 	bcm2835_wr(bs, BCM2835_SPI_DLEN, bs->tx_len);  in bcm2835_spi_transfer_one_dma()
791 	bs->tx_dma_active = true;  in bcm2835_spi_transfer_one_dma()
795 	dma_async_issue_pending(ctlr->dma_tx);  in bcm2835_spi_transfer_one_dma()
797 	/* setup rx-DMA late - to run transfers while  in bcm2835_spi_transfer_one_dma()
798 	 * mapping of the rx buffers still takes place  in bcm2835_spi_transfer_one_dma()
801 	if (bs->rx_buf) {  in bcm2835_spi_transfer_one_dma()
804 		cookie = dmaengine_submit(bs->clear_rx_desc[spi->chip_select]);  in bcm2835_spi_transfer_one_dma()
809 		dmaengine_terminate_sync(ctlr->dma_tx);  in bcm2835_spi_transfer_one_dma()
810 		bs->tx_dma_active = false;  in bcm2835_spi_transfer_one_dma()
814 	/* start rx dma late */  in bcm2835_spi_transfer_one_dma()
815 	dma_async_issue_pending(ctlr->dma_rx);  in bcm2835_spi_transfer_one_dma()
816 	bs->rx_dma_active = true;  in bcm2835_spi_transfer_one_dma()
820 	 * In case of a very short TX-only transfer, bcm2835_spi_dma_tx_done()  in bcm2835_spi_transfer_one_dma()
821 	 * may run before RX DMA is issued.  Terminate RX DMA if so.  in bcm2835_spi_transfer_one_dma()
823 	if (!bs->rx_buf && !bs->tx_dma_active &&  in bcm2835_spi_transfer_one_dma()
824 	    cmpxchg(&bs->rx_dma_active, true, false)) {  in bcm2835_spi_transfer_one_dma()
825 		dmaengine_terminate_async(ctlr->dma_rx);  in bcm2835_spi_transfer_one_dma()
843 	if (tfr->len < BCM2835_SPI_DMA_MIN_LENGTH)  in bcm2835_spi_can_dma()
855 	if (ctlr->dma_tx) {  in bcm2835_dma_release()
856 		dmaengine_terminate_sync(ctlr->dma_tx);  in bcm2835_dma_release()
858 		if (bs->fill_tx_desc)  in bcm2835_dma_release()
859 			dmaengine_desc_free(bs->fill_tx_desc);  in bcm2835_dma_release()
861 		if (bs->fill_tx_addr)  in bcm2835_dma_release()
862 			dma_unmap_page_attrs(ctlr->dma_tx->device->dev,  in bcm2835_dma_release()
863 					     bs->fill_tx_addr, sizeof(u32),  in bcm2835_dma_release()
867 		dma_release_channel(ctlr->dma_tx);  in bcm2835_dma_release()
868 		ctlr->dma_tx = NULL;  in bcm2835_dma_release()
871 	if (ctlr->dma_rx) {  in bcm2835_dma_release()
872 		dmaengine_terminate_sync(ctlr->dma_rx);  in bcm2835_dma_release()
875 			if (bs->clear_rx_desc[i])  in bcm2835_dma_release()
876 				dmaengine_desc_free(bs->clear_rx_desc[i]);  in bcm2835_dma_release()
878 		if (bs->clear_rx_addr)  in bcm2835_dma_release()
879 			dma_unmap_single(ctlr->dma_rx->device->dev,  in bcm2835_dma_release()
880 					 bs->clear_rx_addr,  in bcm2835_dma_release()
881 					 sizeof(bs->clear_rx_cs),  in bcm2835_dma_release()
884 		dma_release_channel(ctlr->dma_rx);  in bcm2835_dma_release()
885 		ctlr->dma_rx = NULL;  in bcm2835_dma_release()
897 	/* base address in dma-space */  in bcm2835_dma_init()
898 	addr = of_get_address(ctlr->dev.of_node, 0, NULL, NULL);  in bcm2835_dma_init()
900 		dev_err(dev, "could not get DMA-register address - not using dma mode\n");  in bcm2835_dma_init()
906 	/* get tx/rx dma */  in bcm2835_dma_init()
907 	ctlr->dma_tx = dma_request_chan(dev, "tx");  in bcm2835_dma_init()
908 	if (IS_ERR(ctlr->dma_tx)) {  in bcm2835_dma_init()
909 		dev_err(dev, "no tx-dma configuration found - not using dma mode\n");  in bcm2835_dma_init()
910 		ret = PTR_ERR(ctlr->dma_tx);  in bcm2835_dma_init()
911 		ctlr->dma_tx = NULL;  in bcm2835_dma_init()
914 	ctlr->dma_rx = dma_request_chan(dev, "rx");  in bcm2835_dma_init()
915 	if (IS_ERR(ctlr->dma_rx)) {  in bcm2835_dma_init()
916 		dev_err(dev, "no rx-dma configuration found - not using dma mode\n");  in bcm2835_dma_init()
917 		ret = PTR_ERR(ctlr->dma_rx);  in bcm2835_dma_init()
918 		ctlr->dma_rx = NULL;  in bcm2835_dma_init()
923 	 * The TX DMA channel either copies a transfer's TX buffer to the FIFO  in bcm2835_dma_init()
924 	 * or, in case of an RX-only transfer, cyclically copies from the zero  in bcm2835_dma_init()
925 	 * page to the FIFO using a preallocated, reusable descriptor.  in bcm2835_dma_init()
930 	ret = dmaengine_slave_config(ctlr->dma_tx, &slave_config);  in bcm2835_dma_init()
934 	bs->fill_tx_addr = dma_map_page_attrs(ctlr->dma_tx->device->dev,  in bcm2835_dma_init()
938 	if (dma_mapping_error(ctlr->dma_tx->device->dev, bs->fill_tx_addr)) {  in bcm2835_dma_init()
939 		dev_err(dev, "cannot map zero page - not using DMA mode\n");  in bcm2835_dma_init()
940 		bs->fill_tx_addr = 0;  in bcm2835_dma_init()
941 		ret = -ENOMEM;  in bcm2835_dma_init()
945 	bs->fill_tx_desc = dmaengine_prep_dma_cyclic(ctlr->dma_tx,  in bcm2835_dma_init()
946 						     bs->fill_tx_addr,  in bcm2835_dma_init()
949 	if (!bs->fill_tx_desc) {  in bcm2835_dma_init()
950 		dev_err(dev, "cannot prepare fill_tx_desc - not using DMA mode\n");  in bcm2835_dma_init()
951 		ret = -ENOMEM;  in bcm2835_dma_init()
955 	ret = dmaengine_desc_set_reuse(bs->fill_tx_desc);  in bcm2835_dma_init()
957 		dev_err(dev, "cannot reuse fill_tx_desc - not using DMA mode\n");  in bcm2835_dma_init()
962 	 * The RX DMA channel is used bidirectionally:  It either reads the  in bcm2835_dma_init()
963 	 * RX FIFO or, in case of a TX-only transfer, cyclically writes a  in bcm2835_dma_init()
964 	 * precalculated value to the CS register to clear the RX FIFO.  in bcm2835_dma_init()
971 	ret = dmaengine_slave_config(ctlr->dma_rx, &slave_config);  in bcm2835_dma_init()
975 	bs->clear_rx_addr = dma_map_single(ctlr->dma_rx->device->dev,  in bcm2835_dma_init()
976 					   bs->clear_rx_cs,  in bcm2835_dma_init()
977 					   sizeof(bs->clear_rx_cs),  in bcm2835_dma_init()
979 	if (dma_mapping_error(ctlr->dma_rx->device->dev, bs->clear_rx_addr)) {  in bcm2835_dma_init()
980 		dev_err(dev, "cannot map clear_rx_cs - not using DMA mode\n");  in bcm2835_dma_init()
981 		bs->clear_rx_addr = 0;  in bcm2835_dma_init()
982 		ret = -ENOMEM;  in bcm2835_dma_init()
987 		bs->clear_rx_desc[i] = dmaengine_prep_dma_cyclic(ctlr->dma_rx,  in bcm2835_dma_init()
988 					   bs->clear_rx_addr + i * sizeof(u32),  in bcm2835_dma_init()
991 		if (!bs->clear_rx_desc[i]) {  in bcm2835_dma_init()
992 			dev_err(dev, "cannot prepare clear_rx_desc - not using DMA mode\n");  in bcm2835_dma_init()
993 			ret = -ENOMEM;  in bcm2835_dma_init()
997 		ret = dmaengine_desc_set_reuse(bs->clear_rx_desc[i]);  in bcm2835_dma_init()
999 			dev_err(dev, "cannot reuse clear_rx_desc - not using DMA mode\n");  in bcm2835_dma_init()
1005 	ctlr->can_dma = bcm2835_spi_can_dma;  in bcm2835_dma_init()
1010 	dev_err(dev, "issue configuring dma: %d - not using DMA mode\n",  in bcm2835_dma_init()
1019 	if (ret != -EPROBE_DEFER)  in bcm2835_dma_init()
1034 	bs->count_transfer_polling++;  in bcm2835_spi_transfer_one_poll()
1039 	/* fill in the fifo before timeout calculations  in bcm2835_spi_transfer_one_poll()
1049 	while (bs->rx_len) {  in bcm2835_spi_transfer_one_poll()
1050 		/* fill in tx fifo with remaining data */  in bcm2835_spi_transfer_one_poll()
1053 		/* read from fifo as much as possible */  in bcm2835_spi_transfer_one_poll()
1059 		if (bs->rx_len && time_after(jiffies, timeout)) {  in bcm2835_spi_transfer_one_poll()
1060 			dev_dbg_ratelimited(&spi->dev,  in bcm2835_spi_transfer_one_poll()
1061 …  "timeout period reached: jiffies: %lu remaining tx/rx: %d/%d - falling back to interrupt mode\n",  in bcm2835_spi_transfer_one_poll()
1062 					    jiffies - timeout,  in bcm2835_spi_transfer_one_poll()
1063 					    bs->tx_len, bs->rx_len);  in bcm2835_spi_transfer_one_poll()
1067 			bs->count_transfer_irq_after_polling++;  in bcm2835_spi_transfer_one_poll()
1074 	/* Transfer complete - reset SPI HW */  in bcm2835_spi_transfer_one_poll()
1087 	u32 cs = bs->prepare_cs[spi->chip_select];  in bcm2835_spi_transfer_one()
1090 	spi_hz = tfr->speed_hz;  in bcm2835_spi_transfer_one()
1091 	clk_hz = clk_get_rate(bs->clk);  in bcm2835_spi_transfer_one()
1105 	tfr->effective_speed_hz = cdiv ? (clk_hz / cdiv) : (clk_hz / 65536);  in bcm2835_spi_transfer_one()
1108 	/* handle all the 3-wire mode */  in bcm2835_spi_transfer_one()
1109 	if (spi->mode & SPI_3WIRE && tfr->rx_buf)  in bcm2835_spi_transfer_one()
1113 	bs->tx_buf = tfr->tx_buf;  in bcm2835_spi_transfer_one()
1114 	bs->rx_buf = tfr->rx_buf;  in bcm2835_spi_transfer_one()
1115 	bs->tx_len = tfr->len;  in bcm2835_spi_transfer_one()
1116 	bs->rx_len = tfr->len;  in bcm2835_spi_transfer_one()
1121 	 * per byte per polling limit.  E.g., we can transfer 1 byte in 30 us  in bcm2835_spi_transfer_one()
1125 	byte_limit = hz_per_byte ? tfr->effective_speed_hz / hz_per_byte : 1;  in bcm2835_spi_transfer_one()
1128 	if (tfr->len < byte_limit)  in bcm2835_spi_transfer_one()
1135 	if (ctlr->can_dma && bcm2835_spi_can_dma(ctlr, spi, tfr))  in bcm2835_spi_transfer_one()
1138 	/* run in interrupt-mode */  in bcm2835_spi_transfer_one()
1145 	struct spi_device *spi = msg->spi;  in bcm2835_spi_prepare_message()
1149 	if (ctlr->can_dma) {  in bcm2835_spi_prepare_message()
1152 		 * the SPI HW due to DLEN. Split up transfers (32-bit FIFO  in bcm2835_spi_prepare_message()
1165 	bcm2835_wr(bs, BCM2835_SPI_CS, bs->prepare_cs[spi->chip_select]);  in bcm2835_spi_prepare_message()
1176 	dmaengine_terminate_sync(ctlr->dma_tx);  in bcm2835_spi_handle_err()
1177 	bs->tx_dma_active = false;  in bcm2835_spi_handle_err()
1178 	dmaengine_terminate_sync(ctlr->dma_rx);  in bcm2835_spi_handle_err()
1179 	bs->rx_dma_active = false;  in bcm2835_spi_handle_err()
1188 	return !strcmp(chip->label, data);  in chip_match_name()
1193 	struct spi_controller *ctlr = spi->controller;  in bcm2835_spi_setup()
1199 	 * Precalculate SPI slave's CS register value for ->prepare_message():  in bcm2835_spi_setup()
1200 	 * The driver always uses software-controlled GPIO chip select, hence  in bcm2835_spi_setup()
1201 	 * set the hardware-controlled native chip select to an invalid value  in bcm2835_spi_setup()
1205 	if (spi->mode & SPI_CPOL)  in bcm2835_spi_setup()
1207 	if (spi->mode & SPI_CPHA)  in bcm2835_spi_setup()
1209 	bs->prepare_cs[spi->chip_select] = cs;  in bcm2835_spi_setup()
1212 	 * Precalculate SPI slave's CS register value to clear RX FIFO  in bcm2835_spi_setup()
1213 	 * in case of a TX-only DMA transfer.  in bcm2835_spi_setup()
1215 	if (ctlr->dma_rx) {  in bcm2835_spi_setup()
1216 		bs->clear_rx_cs[spi->chip_select] = cs |  in bcm2835_spi_setup()
1220 		dma_sync_single_for_device(ctlr->dma_rx->device->dev,  in bcm2835_spi_setup()
1221 					   bs->clear_rx_addr,  in bcm2835_spi_setup()
1222 					   sizeof(bs->clear_rx_cs),  in bcm2835_spi_setup()
1227 	 * sanity checking the native-chipselects  in bcm2835_spi_setup()
1229 	if (spi->mode & SPI_NO_CS)  in bcm2835_spi_setup()
1235 	if (spi->cs_gpiod)  in bcm2835_spi_setup()
1237 	if (spi->chip_select > 1) {  in bcm2835_spi_setup()
1242 		dev_err(&spi->dev,  in bcm2835_spi_setup()
1243 			"setup: only two native chip-selects are supported\n");  in bcm2835_spi_setup()
1244 		return -EINVAL;  in bcm2835_spi_setup()
1252 	 * and fix it. Why is the GPIO descriptor in spi->cs_gpiod  in bcm2835_spi_setup()
1257 	chip = gpiochip_find("pinctrl-bcm2835", chip_match_name);  in bcm2835_spi_setup()
1261 	spi->cs_gpiod = gpiochip_request_own_desc(chip, 8 - spi->chip_select,  in bcm2835_spi_setup()
1265 	if (IS_ERR(spi->cs_gpiod))  in bcm2835_spi_setup()
1266 		return PTR_ERR(spi->cs_gpiod);  in bcm2835_spi_setup()
1269 	dev_info(&spi->dev, "setting up native-CS%i to use GPIO\n",  in bcm2835_spi_setup()
1270 		 spi->chip_select);  in bcm2835_spi_setup()
1281 	ctlr = devm_spi_alloc_master(&pdev->dev, ALIGN(sizeof(*bs),  in bcm2835_spi_probe()
1284 		return -ENOMEM;  in bcm2835_spi_probe()
1288 	ctlr->use_gpio_descriptors = true;  in bcm2835_spi_probe()
1289 	ctlr->mode_bits = BCM2835_SPI_MODE_BITS;  in bcm2835_spi_probe()
1290 	ctlr->bits_per_word_mask = SPI_BPW_MASK(8);  in bcm2835_spi_probe()
1291 	ctlr->num_chipselect = BCM2835_SPI_NUM_CS;  in bcm2835_spi_probe()
1292 	ctlr->setup = bcm2835_spi_setup;  in bcm2835_spi_probe()
1293 	ctlr->transfer_one = bcm2835_spi_transfer_one;  in bcm2835_spi_probe()
1294 	ctlr->handle_err = bcm2835_spi_handle_err;  in bcm2835_spi_probe()
1295 	ctlr->prepare_message = bcm2835_spi_prepare_message;  in bcm2835_spi_probe()
1296 	ctlr->dev.of_node = pdev->dev.of_node;  in bcm2835_spi_probe()
1299 	bs->ctlr = ctlr;  in bcm2835_spi_probe()
1301 	bs->regs = devm_platform_ioremap_resource(pdev, 0);  in bcm2835_spi_probe()
1302 	if (IS_ERR(bs->regs))  in bcm2835_spi_probe()
1303 		return PTR_ERR(bs->regs);  in bcm2835_spi_probe()
1305 	bs->clk = devm_clk_get(&pdev->dev, NULL);  in bcm2835_spi_probe()
1306 	if (IS_ERR(bs->clk))  in bcm2835_spi_probe()
1307 		return dev_err_probe(&pdev->dev, PTR_ERR(bs->clk),  in bcm2835_spi_probe()
1310 	bs->irq = platform_get_irq(pdev, 0);  in bcm2835_spi_probe()
1311 	if (bs->irq <= 0)  in bcm2835_spi_probe()
1312 		return bs->irq ? bs->irq : -ENODEV;  in bcm2835_spi_probe()
1314 	clk_prepare_enable(bs->clk);  in bcm2835_spi_probe()
1316 	err = bcm2835_dma_init(ctlr, &pdev->dev, bs);  in bcm2835_spi_probe()
1324 	err = devm_request_irq(&pdev->dev, bs->irq, bcm2835_spi_interrupt, 0,  in bcm2835_spi_probe()
1325 			       dev_name(&pdev->dev), bs);  in bcm2835_spi_probe()
1327 		dev_err(&pdev->dev, "could not request IRQ: %d\n", err);  in bcm2835_spi_probe()
1333 		dev_err(&pdev->dev, "could not register SPI controller: %d\n",  in bcm2835_spi_probe()
1338 	bcm2835_debugfs_create(bs, dev_name(&pdev->dev));  in bcm2835_spi_probe()
1345 	clk_disable_unprepare(bs->clk);  in bcm2835_spi_probe()
1364 	clk_disable_unprepare(bs->clk);  in bcm2835_spi_remove()
1375 		dev_err(&pdev->dev, "failed to shutdown\n");  in bcm2835_spi_shutdown()
1379 	{ .compatible = "brcm,bcm2835-spi", },