1 // SPDX-License-Identifier: GPL-2.0-only
3  * Driver for Broadcom BRCMSTB, NSP,  NS2, Cygnus SPI Controllers
21 #include <linux/spi/spi.h>
22 #include <linux/spi/spi-mem.h>
25 #include "spi-bcm-qspi.h"
171  * to TXRAM and RXRAM when used as 32-bit registers respectively
255 static inline bool has_bspi(struct bcm_qspi *qspi)  in has_bspi()  argument
257 	return qspi->bspi_mode;  in has_bspi()
260 /* hardware supports spcr3 and fast baud-rate  */
261 static inline bool bcm_qspi_has_fastbr(struct bcm_qspi *qspi)  in bcm_qspi_has_fastbr()  argument
263 	if (!has_bspi(qspi) &&  in bcm_qspi_has_fastbr()
264 	    ((qspi->mspi_maj_rev >= 1) &&  in bcm_qspi_has_fastbr()
265 	     (qspi->mspi_min_rev >= 5)))  in bcm_qspi_has_fastbr()
272 static inline bool bcm_qspi_has_sysclk_108(struct bcm_qspi *qspi)  in bcm_qspi_has_sysclk_108()  argument
274 	if (!has_bspi(qspi) && (qspi->mspi_spcr3_sysclk ||  in bcm_qspi_has_sysclk_108()
275 	    ((qspi->mspi_maj_rev >= 1) &&  in bcm_qspi_has_sysclk_108()
276 	     (qspi->mspi_min_rev >= 6))))  in bcm_qspi_has_sysclk_108()
282 static inline int bcm_qspi_spbr_min(struct bcm_qspi *qspi)  in bcm_qspi_spbr_min()  argument
284 	if (bcm_qspi_has_fastbr(qspi))  in bcm_qspi_spbr_min()
285 		return (bcm_qspi_has_sysclk_108(qspi) ? 4 : 1);  in bcm_qspi_spbr_min()
296 	if (xp->speed_hz)  in bcm_qspi_calc_spbr()
297 		spbr = clk_speed_hz / (xp->speed_hz * 2);  in bcm_qspi_calc_spbr()
302 /* Read qspi controller register*/
303 static inline u32 bcm_qspi_read(struct bcm_qspi *qspi, enum base_type type,  in bcm_qspi_read()  argument
306 	return bcm_qspi_readl(qspi->big_endian, qspi->base[type] + offset);  in bcm_qspi_read()
309 /* Write qspi controller register*/
310 static inline void bcm_qspi_write(struct bcm_qspi *qspi, enum base_type type,  in bcm_qspi_write()  argument
313 	bcm_qspi_writel(qspi->big_endian, data, qspi->base[type] + offset);  in bcm_qspi_write()
317 static int bcm_qspi_bspi_busy_poll(struct bcm_qspi *qspi)  in bcm_qspi_bspi_busy_poll()  argument
323 		if (!(bcm_qspi_read(qspi, BSPI, BSPI_BUSY_STATUS) & 1))  in bcm_qspi_bspi_busy_poll()
327 	dev_warn(&qspi->pdev->dev, "timeout waiting for !busy_status\n");  in bcm_qspi_bspi_busy_poll()
328 	return -EIO;  in bcm_qspi_bspi_busy_poll()
331 static inline bool bcm_qspi_bspi_ver_three(struct bcm_qspi *qspi)  in bcm_qspi_bspi_ver_three()  argument
333 	if (qspi->bspi_maj_rev < 4)  in bcm_qspi_bspi_ver_three()
338 static void bcm_qspi_bspi_flush_prefetch_buffers(struct bcm_qspi *qspi)  in bcm_qspi_bspi_flush_prefetch_buffers()  argument
340 	bcm_qspi_bspi_busy_poll(qspi);  in bcm_qspi_bspi_flush_prefetch_buffers()
342 	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 1);  in bcm_qspi_bspi_flush_prefetch_buffers()
343 	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 1);  in bcm_qspi_bspi_flush_prefetch_buffers()
344 	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 0);  in bcm_qspi_bspi_flush_prefetch_buffers()
345 	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 0);  in bcm_qspi_bspi_flush_prefetch_buffers()
348 static int bcm_qspi_bspi_lr_is_fifo_empty(struct bcm_qspi *qspi)  in bcm_qspi_bspi_lr_is_fifo_empty()  argument
350 	return (bcm_qspi_read(qspi, BSPI, BSPI_RAF_STATUS) &  in bcm_qspi_bspi_lr_is_fifo_empty()
354 static inline u32 bcm_qspi_bspi_lr_read_fifo(struct bcm_qspi *qspi)  in bcm_qspi_bspi_lr_read_fifo()  argument
356 	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_RAF_READ_DATA);  in bcm_qspi_bspi_lr_read_fifo()
359 	if (bcm_qspi_bspi_ver_three(qspi))  in bcm_qspi_bspi_lr_read_fifo()
365 static inline void bcm_qspi_bspi_lr_start(struct bcm_qspi *qspi)  in bcm_qspi_bspi_lr_start()  argument
367 	bcm_qspi_bspi_busy_poll(qspi);  in bcm_qspi_bspi_lr_start()
368 	bcm_qspi_write(qspi, BSPI, BSPI_RAF_CTRL,  in bcm_qspi_bspi_lr_start()
372 static inline void bcm_qspi_bspi_lr_clear(struct bcm_qspi *qspi)  in bcm_qspi_bspi_lr_clear()  argument
374 	bcm_qspi_write(qspi, BSPI, BSPI_RAF_CTRL,  in bcm_qspi_bspi_lr_clear()
376 	bcm_qspi_bspi_flush_prefetch_buffers(qspi);  in bcm_qspi_bspi_lr_clear()
379 static void bcm_qspi_bspi_lr_data_read(struct bcm_qspi *qspi)  in bcm_qspi_bspi_lr_data_read()  argument
381 	u32 *buf = (u32 *)qspi->bspi_rf_op->data.buf.in;  in bcm_qspi_bspi_lr_data_read()
384 	dev_dbg(&qspi->pdev->dev, "xfer %p rx %p rxlen %d\n", qspi->bspi_rf_op,  in bcm_qspi_bspi_lr_data_read()
385 		qspi->bspi_rf_op->data.buf.in, qspi->bspi_rf_op_len);  in bcm_qspi_bspi_lr_data_read()
386 	while (!bcm_qspi_bspi_lr_is_fifo_empty(qspi)) {  in bcm_qspi_bspi_lr_data_read()
387 		data = bcm_qspi_bspi_lr_read_fifo(qspi);  in bcm_qspi_bspi_lr_data_read()
388 		if (likely(qspi->bspi_rf_op_len >= 4) &&  in bcm_qspi_bspi_lr_data_read()
390 			buf[qspi->bspi_rf_op_idx++] = data;  in bcm_qspi_bspi_lr_data_read()
391 			qspi->bspi_rf_op_len -= 4;  in bcm_qspi_bspi_lr_data_read()
394 			u8 *cbuf = (u8 *)&buf[qspi->bspi_rf_op_idx];  in bcm_qspi_bspi_lr_data_read()
397 			while (qspi->bspi_rf_op_len) {  in bcm_qspi_bspi_lr_data_read()
400 				qspi->bspi_rf_op_len--;  in bcm_qspi_bspi_lr_data_read()
406 static void bcm_qspi_bspi_set_xfer_params(struct bcm_qspi *qspi, u8 cmd_byte,  in bcm_qspi_bspi_set_xfer_params()  argument
409 	bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, 0);  in bcm_qspi_bspi_set_xfer_params()
410 	bcm_qspi_write(qspi, BSPI, BSPI_BITS_PER_CYCLE, bpc);  in bcm_qspi_bspi_set_xfer_params()
411 	bcm_qspi_write(qspi, BSPI, BSPI_BITS_PER_PHASE, bpp);  in bcm_qspi_bspi_set_xfer_params()
412 	bcm_qspi_write(qspi, BSPI, BSPI_CMD_AND_MODE_BYTE, cmd_byte);  in bcm_qspi_bspi_set_xfer_params()
413 	bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, flex_mode);  in bcm_qspi_bspi_set_xfer_params()
416 static int bcm_qspi_bspi_set_flex_mode(struct bcm_qspi *qspi,  in bcm_qspi_bspi_set_flex_mode()  argument
420 	u8 command = op->cmd.opcode;  in bcm_qspi_bspi_set_flex_mode()
421 	int width = op->data.buswidth ? op->data.buswidth : SPI_NBITS_SINGLE;  in bcm_qspi_bspi_set_flex_mode()
422 	int addrlen = op->addr.nbytes;  in bcm_qspi_bspi_set_flex_mode()
425 	dev_dbg(&qspi->pdev->dev, "set flex mode w %x addrlen %x hp %d\n",  in bcm_qspi_bspi_set_flex_mode()
431 	if (op->dummy.nbytes)  in bcm_qspi_bspi_set_flex_mode()
432 		bpp |= (op->dummy.nbytes * 8) / op->dummy.buswidth;  in bcm_qspi_bspi_set_flex_mode()
443 			bpc |= 0x00010100; /* address and mode are 2-bit */  in bcm_qspi_bspi_set_flex_mode()
450 			bpc |= 0x00020200; /* address and mode are 4-bit */  in bcm_qspi_bspi_set_flex_mode()
455 		return -EINVAL;  in bcm_qspi_bspi_set_flex_mode()
458 	bcm_qspi_bspi_set_xfer_params(qspi, command, bpp, bpc, flex_mode);  in bcm_qspi_bspi_set_flex_mode()
463 static int bcm_qspi_bspi_set_override(struct bcm_qspi *qspi,  in bcm_qspi_bspi_set_override()  argument
466 	int width = op->data.buswidth ? op->data.buswidth : SPI_NBITS_SINGLE;  in bcm_qspi_bspi_set_override()
467 	int addrlen = op->addr.nbytes;  in bcm_qspi_bspi_set_override()
468 	u32 data = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);  in bcm_qspi_bspi_set_override()
470 	dev_dbg(&qspi->pdev->dev, "set override mode w %x addrlen %x hp %d\n",  in bcm_qspi_bspi_set_override()
490 		return -EINVAL;  in bcm_qspi_bspi_set_override()
502 	bcm_qspi_write(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL, data);  in bcm_qspi_bspi_set_override()
503 	bcm_qspi_bspi_set_xfer_params(qspi, op->cmd.opcode, 0, 0, 0);  in bcm_qspi_bspi_set_override()
508 static int bcm_qspi_bspi_set_mode(struct bcm_qspi *qspi,  in bcm_qspi_bspi_set_mode()  argument
512 	int width = op->data.buswidth ? op->data.buswidth : SPI_NBITS_SINGLE;  in bcm_qspi_bspi_set_mode()
513 	int addrlen = op->addr.nbytes;  in bcm_qspi_bspi_set_mode()
516 	qspi->xfer_mode.flex_mode = true;  in bcm_qspi_bspi_set_mode()
518 	if (!bcm_qspi_bspi_ver_three(qspi)) {  in bcm_qspi_bspi_set_mode()
521 		val = bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);  in bcm_qspi_bspi_set_mode()
523 		if (val & mask || qspi->s3_strap_override_ctrl & mask) {  in bcm_qspi_bspi_set_mode()
524 			qspi->xfer_mode.flex_mode = false;  in bcm_qspi_bspi_set_mode()
525 			bcm_qspi_write(qspi, BSPI, BSPI_FLEX_MODE_ENABLE, 0);  in bcm_qspi_bspi_set_mode()
526 			error = bcm_qspi_bspi_set_override(qspi, op, hp);  in bcm_qspi_bspi_set_mode()
530 	if (qspi->xfer_mode.flex_mode)  in bcm_qspi_bspi_set_mode()
531 		error = bcm_qspi_bspi_set_flex_mode(qspi, op, hp);  in bcm_qspi_bspi_set_mode()
534 		dev_warn(&qspi->pdev->dev,  in bcm_qspi_bspi_set_mode()
537 	} else if (qspi->xfer_mode.width != width ||  in bcm_qspi_bspi_set_mode()
538 		   qspi->xfer_mode.addrlen != addrlen ||  in bcm_qspi_bspi_set_mode()
539 		   qspi->xfer_mode.hp != hp) {  in bcm_qspi_bspi_set_mode()
540 		qspi->xfer_mode.width = width;  in bcm_qspi_bspi_set_mode()
541 		qspi->xfer_mode.addrlen = addrlen;  in bcm_qspi_bspi_set_mode()
542 		qspi->xfer_mode.hp = hp;  in bcm_qspi_bspi_set_mode()
543 		dev_dbg(&qspi->pdev->dev,  in bcm_qspi_bspi_set_mode()
544 			"cs:%d %d-lane output, %d-byte address%s\n",  in bcm_qspi_bspi_set_mode()
545 			qspi->curr_cs,  in bcm_qspi_bspi_set_mode()
546 			qspi->xfer_mode.width,  in bcm_qspi_bspi_set_mode()
547 			qspi->xfer_mode.addrlen,  in bcm_qspi_bspi_set_mode()
548 			qspi->xfer_mode.hp != -1 ? ", hp mode" : "");  in bcm_qspi_bspi_set_mode()
554 static void bcm_qspi_enable_bspi(struct bcm_qspi *qspi)  in bcm_qspi_enable_bspi()  argument
556 	if (!has_bspi(qspi))  in bcm_qspi_enable_bspi()
559 	qspi->bspi_enabled = 1;  in bcm_qspi_enable_bspi()
560 	if ((bcm_qspi_read(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL) & 1) == 0)  in bcm_qspi_enable_bspi()
563 	bcm_qspi_bspi_flush_prefetch_buffers(qspi);  in bcm_qspi_enable_bspi()
565 	bcm_qspi_write(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL, 0);  in bcm_qspi_enable_bspi()
569 static void bcm_qspi_disable_bspi(struct bcm_qspi *qspi)  in bcm_qspi_disable_bspi()  argument
571 	if (!has_bspi(qspi))  in bcm_qspi_disable_bspi()
574 	qspi->bspi_enabled = 0;  in bcm_qspi_disable_bspi()
575 	if ((bcm_qspi_read(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL) & 1))  in bcm_qspi_disable_bspi()
578 	bcm_qspi_bspi_busy_poll(qspi);  in bcm_qspi_disable_bspi()
579 	bcm_qspi_write(qspi, BSPI, BSPI_MAST_N_BOOT_CTRL, 1);  in bcm_qspi_disable_bspi()
583 static void bcm_qspi_chip_select(struct bcm_qspi *qspi, int cs)  in bcm_qspi_chip_select()  argument
588 	if (cs >= 0 && qspi->base[CHIP_SELECT]) {  in bcm_qspi_chip_select()
589 		rd = bcm_qspi_read(qspi, CHIP_SELECT, 0);  in bcm_qspi_chip_select()
593 		bcm_qspi_write(qspi, CHIP_SELECT, 0, wr);  in bcm_qspi_chip_select()
597 	dev_dbg(&qspi->pdev->dev, "using cs:%d\n", cs);  in bcm_qspi_chip_select()
598 	qspi->curr_cs = cs;  in bcm_qspi_chip_select()
604 	return (cur->speed_hz != prev->speed_hz) ||  in bcmspi_parms_did_change()
605 		(cur->mode != prev->mode) ||  in bcmspi_parms_did_change()
606 		(cur->bits_per_word != prev->bits_per_word);  in bcmspi_parms_did_change()
611 static void bcm_qspi_hw_set_parms(struct bcm_qspi *qspi,  in bcm_qspi_hw_set_parms()  argument
616 	if (!bcmspi_parms_did_change(xp, &qspi->last_parms))  in bcm_qspi_hw_set_parms()
619 	if (!qspi->mspi_maj_rev)  in bcm_qspi_hw_set_parms()
632 	if (xp->bits_per_word != 16 && xp->bits_per_word != 64)  in bcm_qspi_hw_set_parms()
633 		spcr |= xp->bits_per_word << MSPI_SPCR0_MSB_BITS_SHIFT;  in bcm_qspi_hw_set_parms()
635 	spcr |= xp->mode & (MSPI_SPCR0_MSB_CPHA | MSPI_SPCR0_MSB_CPOL);  in bcm_qspi_hw_set_parms()
636 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR0_MSB, spcr);  in bcm_qspi_hw_set_parms()
638 	if (bcm_qspi_has_fastbr(qspi)) {  in bcm_qspi_hw_set_parms()
644 		if (xp->mode & SPI_3WIRE)  in bcm_qspi_hw_set_parms()
647 		if (bcm_qspi_has_sysclk_108(qspi)) {  in bcm_qspi_hw_set_parms()
653 				qspi->base_clk = MSPI_BASE_FREQ;  in bcm_qspi_hw_set_parms()
657 				qspi->base_clk = MSPI_BASE_FREQ * 4;  in bcm_qspi_hw_set_parms()
661 		if (xp->bits_per_word > 16) {  in bcm_qspi_hw_set_parms()
670 			bcm_qspi_write(qspi, MSPI, MSPI_SPCR1_LSB, 1);  in bcm_qspi_hw_set_parms()
681 		bcm_qspi_write(qspi, MSPI, MSPI_SPCR3, spcr);  in bcm_qspi_hw_set_parms()
685 	qspi->max_speed_hz = qspi->base_clk / (bcm_qspi_spbr_min(qspi) * 2);  in bcm_qspi_hw_set_parms()
686 	spbr = bcm_qspi_calc_spbr(qspi->base_clk, xp);  in bcm_qspi_hw_set_parms()
687 	spbr = clamp_val(spbr, bcm_qspi_spbr_min(qspi), QSPI_SPBR_MAX);  in bcm_qspi_hw_set_parms()
688 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR0_LSB, spbr);  in bcm_qspi_hw_set_parms()
690 	qspi->last_parms = *xp;  in bcm_qspi_hw_set_parms()
693 static void bcm_qspi_update_parms(struct bcm_qspi *qspi,  in bcm_qspi_update_parms()  argument
694 				  struct spi_device *spi,  in bcm_qspi_update_parms()  argument
699 	xp.speed_hz = trans->speed_hz;  in bcm_qspi_update_parms()
700 	xp.bits_per_word = trans->bits_per_word;  in bcm_qspi_update_parms()
701 	xp.mode = spi->mode;  in bcm_qspi_update_parms()
703 	bcm_qspi_hw_set_parms(qspi, &xp);  in bcm_qspi_update_parms()
706 static int bcm_qspi_setup(struct spi_device *spi)  in bcm_qspi_setup()  argument
710 	if (spi->bits_per_word > 64)  in bcm_qspi_setup()
711 		return -EINVAL;  in bcm_qspi_setup()
713 	xp = spi_get_ctldata(spi);  in bcm_qspi_setup()
717 			return -ENOMEM;  in bcm_qspi_setup()
718 		spi_set_ctldata(spi, xp);  in bcm_qspi_setup()
720 	xp->speed_hz = spi->max_speed_hz;  in bcm_qspi_setup()
721 	xp->mode = spi->mode;  in bcm_qspi_setup()
723 	if (spi->bits_per_word)  in bcm_qspi_setup()
724 		xp->bits_per_word = spi->bits_per_word;  in bcm_qspi_setup()
726 		xp->bits_per_word = 8;  in bcm_qspi_setup()
731 static bool bcm_qspi_mspi_transfer_is_last(struct bcm_qspi *qspi,  in bcm_qspi_mspi_transfer_is_last()  argument
734 	if (qt->mspi_last_trans &&  in bcm_qspi_mspi_transfer_is_last()
735 	    spi_transfer_is_last(qspi->host, qt->trans))  in bcm_qspi_mspi_transfer_is_last()
741 static int update_qspi_trans_byte_count(struct bcm_qspi *qspi,  in update_qspi_trans_byte_count()  argument
747 	if (qt->trans->bits_per_word <= 8)  in update_qspi_trans_byte_count()
748 		qt->byte++;  in update_qspi_trans_byte_count()
749 	else if (qt->trans->bits_per_word <= 16)  in update_qspi_trans_byte_count()
750 		qt->byte += 2;  in update_qspi_trans_byte_count()
751 	else if (qt->trans->bits_per_word <= 32)  in update_qspi_trans_byte_count()
752 		qt->byte += 4;  in update_qspi_trans_byte_count()
753 	else if (qt->trans->bits_per_word <= 64)  in update_qspi_trans_byte_count()
754 		qt->byte += 8;  in update_qspi_trans_byte_count()
756 	if (qt->byte >= qt->trans->len) {  in update_qspi_trans_byte_count()
759 		if (qt->trans->delay.value &&  in update_qspi_trans_byte_count()
762 		if (qt->trans->cs_change &&  in update_qspi_trans_byte_count()
766 		if (bcm_qspi_mspi_transfer_is_last(qspi, qt))  in update_qspi_trans_byte_count()
771 		qt->trans = NULL;  in update_qspi_trans_byte_count()
774 	dev_dbg(&qspi->pdev->dev, "trans %p len %d byte %d ret %x\n",  in update_qspi_trans_byte_count()
775 		qt->trans, qt->trans ? qt->trans->len : 0, qt->byte, ret);  in update_qspi_trans_byte_count()
779 static inline u8 read_rxram_slot_u8(struct bcm_qspi *qspi, int slot)  in read_rxram_slot_u8()  argument
784 	return bcm_qspi_read(qspi, MSPI, slot_offset) & 0xff;  in read_rxram_slot_u8()
787 static inline u16 read_rxram_slot_u16(struct bcm_qspi *qspi, int slot)  in read_rxram_slot_u16()  argument
793 	return (bcm_qspi_read(qspi, MSPI, lsb_offset) & 0xff) |  in read_rxram_slot_u16()
794 		((bcm_qspi_read(qspi, MSPI, msb_offset) & 0xff) << 8);  in read_rxram_slot_u16()
797 static inline u32 read_rxram_slot_u32(struct bcm_qspi *qspi, int slot)  in read_rxram_slot_u32()  argument
803 	val = bcm_qspi_read(qspi, MSPI, offset);  in read_rxram_slot_u32()
809 static inline u64 read_rxram_slot_u64(struct bcm_qspi *qspi, int slot)  in read_rxram_slot_u64()  argument
816 	msb = bcm_qspi_read(qspi, MSPI, msb_offset);  in read_rxram_slot_u64()
818 	lsb = bcm_qspi_read(qspi, MSPI, lsb_offset);  in read_rxram_slot_u64()
824 static void read_from_hw(struct bcm_qspi *qspi, int slots)  in read_from_hw()  argument
829 	bcm_qspi_disable_bspi(qspi);  in read_from_hw()
833 		dev_err(&qspi->pdev->dev, "%s: too many slots!\n", __func__);  in read_from_hw()
837 	tp = qspi->trans_pos;  in read_from_hw()
840 		if (tp.trans->bits_per_word <= 8) {  in read_from_hw()
841 			u8 *buf = tp.trans->rx_buf;  in read_from_hw()
844 				buf[tp.byte] = read_rxram_slot_u8(qspi, slot);  in read_from_hw()
845 			dev_dbg(&qspi->pdev->dev, "RD %02x\n",  in read_from_hw()
847 		} else if (tp.trans->bits_per_word <= 16) {  in read_from_hw()
848 			u16 *buf = tp.trans->rx_buf;  in read_from_hw()
851 				buf[tp.byte / 2] = read_rxram_slot_u16(qspi,  in read_from_hw()
853 			dev_dbg(&qspi->pdev->dev, "RD %04x\n",  in read_from_hw()
855 		} else if (tp.trans->bits_per_word <= 32) {  in read_from_hw()
856 			u32 *buf = tp.trans->rx_buf;  in read_from_hw()
859 				buf[tp.byte / 4] = read_rxram_slot_u32(qspi,  in read_from_hw()
861 			dev_dbg(&qspi->pdev->dev, "RD %08x\n",  in read_from_hw()
864 		} else if (tp.trans->bits_per_word <= 64) {  in read_from_hw()
865 			u64 *buf = tp.trans->rx_buf;  in read_from_hw()
868 				buf[tp.byte / 8] = read_rxram_slot_u64(qspi,  in read_from_hw()
870 			dev_dbg(&qspi->pdev->dev, "RD %llx\n",  in read_from_hw()
876 		update_qspi_trans_byte_count(qspi, &tp,  in read_from_hw()
880 	qspi->trans_pos = tp;  in read_from_hw()
883 static inline void write_txram_slot_u8(struct bcm_qspi *qspi, int slot,  in write_txram_slot_u8()  argument
889 	bcm_qspi_write(qspi, MSPI, reg_offset, val);  in write_txram_slot_u8()
892 static inline void write_txram_slot_u16(struct bcm_qspi *qspi, int slot,  in write_txram_slot_u16()  argument
899 	bcm_qspi_write(qspi, MSPI, msb_offset, (val >> 8));  in write_txram_slot_u16()
900 	bcm_qspi_write(qspi, MSPI, lsb_offset, (val & 0xff));  in write_txram_slot_u16()
903 static inline void write_txram_slot_u32(struct bcm_qspi *qspi, int slot,  in write_txram_slot_u32()  argument
909 	bcm_qspi_write(qspi, MSPI, msb_offset, swap4bytes(val));  in write_txram_slot_u32()
912 static inline void write_txram_slot_u64(struct bcm_qspi *qspi, int slot,  in write_txram_slot_u64()  argument
921 	bcm_qspi_write(qspi, MSPI, msb_offset, swap4bytes(msb));  in write_txram_slot_u64()
922 	bcm_qspi_write(qspi, MSPI, lsb_offset, swap4bytes(lsb));  in write_txram_slot_u64()
925 static inline u32 read_cdram_slot(struct bcm_qspi *qspi, int slot)  in read_cdram_slot()  argument
927 	return bcm_qspi_read(qspi, MSPI, MSPI_CDRAM + (slot << 2));  in read_cdram_slot()
930 static inline void write_cdram_slot(struct bcm_qspi *qspi, int slot, u32 val)  in write_cdram_slot()  argument
932 	bcm_qspi_write(qspi, MSPI, (MSPI_CDRAM + (slot << 2)), val);  in write_cdram_slot()
936 static int write_to_hw(struct bcm_qspi *qspi, struct spi_device *spi)  in write_to_hw()  argument
942 	bcm_qspi_disable_bspi(qspi);  in write_to_hw()
943 	tp = qspi->trans_pos;  in write_to_hw()
944 	bcm_qspi_update_parms(qspi, spi, tp.trans);  in write_to_hw()
949 		if (tp.trans->bits_per_word <= 8) {  in write_to_hw()
950 			const u8 *buf = tp.trans->tx_buf;  in write_to_hw()
953 			write_txram_slot_u8(qspi, slot, val);  in write_to_hw()
954 			dev_dbg(&qspi->pdev->dev, "WR %02x\n", val);  in write_to_hw()
955 		} else if (tp.trans->bits_per_word <= 16) {  in write_to_hw()
956 			const u16 *buf = tp.trans->tx_buf;  in write_to_hw()
959 			write_txram_slot_u16(qspi, slot, val);  in write_to_hw()
960 			dev_dbg(&qspi->pdev->dev, "WR %04x\n", val);  in write_to_hw()
961 		} else if (tp.trans->bits_per_word <= 32) {  in write_to_hw()
962 			const u32 *buf = tp.trans->tx_buf;  in write_to_hw()
965 			write_txram_slot_u32(qspi, slot, val);  in write_to_hw()
966 			dev_dbg(&qspi->pdev->dev, "WR %08x\n", val);  in write_to_hw()
967 		} else if (tp.trans->bits_per_word <= 64) {  in write_to_hw()
968 			const u64 *buf = tp.trans->tx_buf;  in write_to_hw()
972 			if (bcm_qspi_has_fastbr(qspi))  in write_to_hw()
975 			write_txram_slot_u64(qspi, slot, val);  in write_to_hw()
976 			dev_dbg(&qspi->pdev->dev, "WR %llx\n", val);  in write_to_hw()
979 		mspi_cdram |= ((tp.trans->bits_per_word <= 8) ? 0 :  in write_to_hw()
983 		if ((spi->mode & SPI_3WIRE) && tp.trans->tx_buf)  in write_to_hw()
986 		if (has_bspi(qspi))  in write_to_hw()
989 			mspi_cdram |= (~(1 << spi_get_chipselect(spi, 0)) &  in write_to_hw()
992 		write_cdram_slot(qspi, slot, mspi_cdram);  in write_to_hw()
994 		tstatus = update_qspi_trans_byte_count(qspi, &tp,  in write_to_hw()
1000 		dev_err(&qspi->pdev->dev, "%s: no data to send?", __func__);  in write_to_hw()
1004 	dev_dbg(&qspi->pdev->dev, "submitting %d slots\n", slot);  in write_to_hw()
1005 	bcm_qspi_write(qspi, MSPI, MSPI_NEWQP, 0);  in write_to_hw()
1006 	bcm_qspi_write(qspi, MSPI, MSPI_ENDQP, slot - 1);  in write_to_hw()
1018 		mspi_cdram = read_cdram_slot(qspi, slot - 1) &  in write_to_hw()
1020 		write_cdram_slot(qspi, slot - 1, mspi_cdram);  in write_to_hw()
1023 	if (has_bspi(qspi))  in write_to_hw()
1024 		bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 1);  in write_to_hw()
1029 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0xe0);  in write_to_hw()
1035 static int bcm_qspi_bspi_exec_mem_op(struct spi_device *spi,  in bcm_qspi_bspi_exec_mem_op()  argument
1038 	struct bcm_qspi *qspi = spi_controller_get_devdata(spi->controller);  in bcm_qspi_bspi_exec_mem_op()  local
1042 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;  in bcm_qspi_bspi_exec_mem_op()
1044 	if (bcm_qspi_bspi_ver_three(qspi))  in bcm_qspi_bspi_exec_mem_op()
1045 		if (op->addr.nbytes == BSPI_ADDRLEN_4BYTES)  in bcm_qspi_bspi_exec_mem_op()
1046 			return -EIO;  in bcm_qspi_bspi_exec_mem_op()
1048 	from = op->addr.val;  in bcm_qspi_bspi_exec_mem_op()
1049 	if (!spi_get_csgpiod(spi, 0))  in bcm_qspi_bspi_exec_mem_op()
1050 		bcm_qspi_chip_select(qspi, spi_get_chipselect(spi, 0));  in bcm_qspi_bspi_exec_mem_op()
1051 	bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);  in bcm_qspi_bspi_exec_mem_op()
1057 	if (!bcm_qspi_bspi_ver_three(qspi)) {  in bcm_qspi_bspi_exec_mem_op()
1059 		bcm_qspi_write(qspi, BSPI,  in bcm_qspi_bspi_exec_mem_op()
1063 	if (!qspi->xfer_mode.flex_mode)  in bcm_qspi_bspi_exec_mem_op()
1068 	if (bcm_qspi_bspi_ver_three(qspi) == true)  in bcm_qspi_bspi_exec_mem_op()
1075 	len = op->data.nbytes;  in bcm_qspi_bspi_exec_mem_op()
1076 	qspi->bspi_rf_op_idx = 0;  in bcm_qspi_bspi_exec_mem_op()
1084 		reinit_completion(&qspi->bspi_done);  in bcm_qspi_bspi_exec_mem_op()
1085 		bcm_qspi_enable_bspi(qspi);  in bcm_qspi_bspi_exec_mem_op()
1087 		qspi->bspi_rf_op = op;  in bcm_qspi_bspi_exec_mem_op()
1088 		qspi->bspi_rf_op_status = 0;  in bcm_qspi_bspi_exec_mem_op()
1089 		qspi->bspi_rf_op_len = rdlen;  in bcm_qspi_bspi_exec_mem_op()
1090 		dev_dbg(&qspi->pdev->dev,  in bcm_qspi_bspi_exec_mem_op()
1092 		bcm_qspi_write(qspi, BSPI, BSPI_RAF_START_ADDR, addr);  in bcm_qspi_bspi_exec_mem_op()
1093 		bcm_qspi_write(qspi, BSPI, BSPI_RAF_NUM_WORDS, len_words);  in bcm_qspi_bspi_exec_mem_op()
1094 		bcm_qspi_write(qspi, BSPI, BSPI_RAF_WATERMARK, 0);  in bcm_qspi_bspi_exec_mem_op()
1095 		if (qspi->soc_intc) {  in bcm_qspi_bspi_exec_mem_op()
1100 			soc_intc->bcm_qspi_int_ack(soc_intc, MSPI_BSPI_DONE);  in bcm_qspi_bspi_exec_mem_op()
1101 			soc_intc->bcm_qspi_int_set(soc_intc, BSPI_DONE, true);  in bcm_qspi_bspi_exec_mem_op()
1106 		bcm_qspi_bspi_lr_start(qspi);  in bcm_qspi_bspi_exec_mem_op()
1107 		if (!wait_for_completion_timeout(&qspi->bspi_done, timeo)) {  in bcm_qspi_bspi_exec_mem_op()
1108 			dev_err(&qspi->pdev->dev, "timeout waiting for BSPI\n");  in bcm_qspi_bspi_exec_mem_op()
1109 			ret = -ETIMEDOUT;  in bcm_qspi_bspi_exec_mem_op()
1115 		len -= rdlen;  in bcm_qspi_bspi_exec_mem_op()
1122 				 struct spi_device *spi,  in bcm_qspi_transfer_one()  argument
1125 	struct bcm_qspi *qspi = spi_controller_get_devdata(host);  in bcm_qspi_transfer_one()  local
1129 	if (!spi_get_csgpiod(spi, 0))  in bcm_qspi_transfer_one()
1130 		bcm_qspi_chip_select(qspi, spi_get_chipselect(spi, 0));  in bcm_qspi_transfer_one()
1131 	qspi->trans_pos.trans = trans;  in bcm_qspi_transfer_one()
1132 	qspi->trans_pos.byte = 0;  in bcm_qspi_transfer_one()
1134 	while (qspi->trans_pos.byte < trans->len) {  in bcm_qspi_transfer_one()
1135 		reinit_completion(&qspi->mspi_done);  in bcm_qspi_transfer_one()
1137 		slots = write_to_hw(qspi, spi);  in bcm_qspi_transfer_one()
1138 		if (!wait_for_completion_timeout(&qspi->mspi_done, timeo)) {  in bcm_qspi_transfer_one()
1139 			dev_err(&qspi->pdev->dev, "timeout waiting for MSPI\n");  in bcm_qspi_transfer_one()
1140 			return -ETIMEDOUT;  in bcm_qspi_transfer_one()
1143 		read_from_hw(qspi, slots);  in bcm_qspi_transfer_one()
1145 	bcm_qspi_enable_bspi(qspi);  in bcm_qspi_transfer_one()
1150 static int bcm_qspi_mspi_exec_mem_op(struct spi_device *spi,  in bcm_qspi_mspi_exec_mem_op()  argument
1153 	struct spi_controller *host = spi->controller;  in bcm_qspi_mspi_exec_mem_op()
1154 	struct bcm_qspi *qspi = spi_controller_get_devdata(host);  in bcm_qspi_mspi_exec_mem_op()  local
1164 	cmd[0] = op->cmd.opcode;  in bcm_qspi_mspi_exec_mem_op()
1165 	for (i = 0; i < op->addr.nbytes; i++)  in bcm_qspi_mspi_exec_mem_op()
1166 		cmd[1 + i] = op->addr.val >> (8 * (op->addr.nbytes - i - 1));  in bcm_qspi_mspi_exec_mem_op()
1169 	t[0].len = op->addr.nbytes + op->dummy.nbytes + 1;  in bcm_qspi_mspi_exec_mem_op()
1170 	t[0].bits_per_word = spi->bits_per_word;  in bcm_qspi_mspi_exec_mem_op()
1171 	t[0].tx_nbits = op->cmd.buswidth;  in bcm_qspi_mspi_exec_mem_op()
1173 	qspi->trans_pos.mspi_last_trans = false;  in bcm_qspi_mspi_exec_mem_op()
1174 	ret = bcm_qspi_transfer_one(host, spi, &t[0]);  in bcm_qspi_mspi_exec_mem_op()
1177 	qspi->trans_pos.mspi_last_trans = true;  in bcm_qspi_mspi_exec_mem_op()
1180 		t[1].rx_buf = op->data.buf.in;  in bcm_qspi_mspi_exec_mem_op()
1181 		t[1].len = op->data.nbytes;  in bcm_qspi_mspi_exec_mem_op()
1182 		t[1].rx_nbits =  op->data.buswidth;  in bcm_qspi_mspi_exec_mem_op()
1183 		t[1].bits_per_word = spi->bits_per_word;  in bcm_qspi_mspi_exec_mem_op()
1184 		ret = bcm_qspi_transfer_one(host, spi, &t[1]);  in bcm_qspi_mspi_exec_mem_op()
1193 	struct spi_device *spi = mem->spi;  in bcm_qspi_exec_mem_op()  local
1194 	struct bcm_qspi *qspi = spi_controller_get_devdata(spi->controller);  in bcm_qspi_exec_mem_op()  local
1200 	if (!op->data.nbytes || !op->addr.nbytes || op->addr.nbytes > 4 ||  in bcm_qspi_exec_mem_op()
1201 	    op->data.dir != SPI_MEM_DATA_IN)  in bcm_qspi_exec_mem_op()
1202 		return -ENOTSUPP;  in bcm_qspi_exec_mem_op()
1204 	buf = op->data.buf.in;  in bcm_qspi_exec_mem_op()
1205 	addr = op->addr.val;  in bcm_qspi_exec_mem_op()
1206 	len = op->data.nbytes;  in bcm_qspi_exec_mem_op()
1208 	if (has_bspi(qspi) && bcm_qspi_bspi_ver_three(qspi) == true) {  in bcm_qspi_exec_mem_op()
1218 		    (~ADDR_4MB_MASK & (addr + len - 1)))  in bcm_qspi_exec_mem_op()
1222 	/* non-aligned and very short transfers are handled by MSPI */  in bcm_qspi_exec_mem_op()
1227 	if (!has_bspi(qspi) || mspi_read)  in bcm_qspi_exec_mem_op()
1228 		return bcm_qspi_mspi_exec_mem_op(spi, op);  in bcm_qspi_exec_mem_op()
1230 	ret = bcm_qspi_bspi_set_mode(qspi, op, 0);  in bcm_qspi_exec_mem_op()
1233 		ret = bcm_qspi_bspi_exec_mem_op(spi, op);  in bcm_qspi_exec_mem_op()
1238 static void bcm_qspi_cleanup(struct spi_device *spi)  in bcm_qspi_cleanup()  argument
1240 	struct bcm_qspi_parms *xp = spi_get_ctldata(spi);  in bcm_qspi_cleanup()
1248 	struct bcm_qspi *qspi = qspi_dev_id->dev;  in bcm_qspi_mspi_l2_isr()  local
1249 	u32 status = bcm_qspi_read(qspi, MSPI, MSPI_MSPI_STATUS);  in bcm_qspi_mspi_l2_isr()
1252 		struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;  in bcm_qspi_mspi_l2_isr()
1255 		bcm_qspi_write(qspi, MSPI, MSPI_MSPI_STATUS, status);  in bcm_qspi_mspi_l2_isr()
1256 		if (qspi->soc_intc)  in bcm_qspi_mspi_l2_isr()
1257 			soc_intc->bcm_qspi_int_ack(soc_intc, MSPI_DONE);  in bcm_qspi_mspi_l2_isr()
1258 		complete(&qspi->mspi_done);  in bcm_qspi_mspi_l2_isr()
1268 	struct bcm_qspi *qspi = qspi_dev_id->dev;  in bcm_qspi_bspi_lr_l2_isr()  local
1269 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;  in bcm_qspi_bspi_lr_l2_isr()
1270 	u32 status = qspi_dev_id->irqp->mask;  in bcm_qspi_bspi_lr_l2_isr()
1272 	if (qspi->bspi_enabled && qspi->bspi_rf_op) {  in bcm_qspi_bspi_lr_l2_isr()
1273 		bcm_qspi_bspi_lr_data_read(qspi);  in bcm_qspi_bspi_lr_l2_isr()
1274 		if (qspi->bspi_rf_op_len == 0) {  in bcm_qspi_bspi_lr_l2_isr()
1275 			qspi->bspi_rf_op = NULL;  in bcm_qspi_bspi_lr_l2_isr()
1276 			if (qspi->soc_intc) {  in bcm_qspi_bspi_lr_l2_isr()
1278 				soc_intc->bcm_qspi_int_set(soc_intc, BSPI_DONE,  in bcm_qspi_bspi_lr_l2_isr()
1284 			if (qspi->bspi_rf_op_status)  in bcm_qspi_bspi_lr_l2_isr()
1285 				bcm_qspi_bspi_lr_clear(qspi);  in bcm_qspi_bspi_lr_l2_isr()
1287 				bcm_qspi_bspi_flush_prefetch_buffers(qspi);  in bcm_qspi_bspi_lr_l2_isr()
1290 		if (qspi->soc_intc)  in bcm_qspi_bspi_lr_l2_isr()
1292 			soc_intc->bcm_qspi_int_ack(soc_intc, BSPI_DONE);  in bcm_qspi_bspi_lr_l2_isr()
1296 	if (qspi->bspi_enabled && status && qspi->bspi_rf_op_len == 0)  in bcm_qspi_bspi_lr_l2_isr()
1297 		complete(&qspi->bspi_done);  in bcm_qspi_bspi_lr_l2_isr()
1305 	struct bcm_qspi *qspi = qspi_dev_id->dev;  in bcm_qspi_bspi_lr_err_l2_isr()  local
1306 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;  in bcm_qspi_bspi_lr_err_l2_isr()
1308 	dev_err(&qspi->pdev->dev, "BSPI INT error\n");  in bcm_qspi_bspi_lr_err_l2_isr()
1309 	qspi->bspi_rf_op_status = -EIO;  in bcm_qspi_bspi_lr_err_l2_isr()
1310 	if (qspi->soc_intc)  in bcm_qspi_bspi_lr_err_l2_isr()
1312 		soc_intc->bcm_qspi_int_ack(soc_intc, BSPI_ERR);  in bcm_qspi_bspi_lr_err_l2_isr()
1314 	complete(&qspi->bspi_done);  in bcm_qspi_bspi_lr_err_l2_isr()
1321 	struct bcm_qspi *qspi = qspi_dev_id->dev;  in bcm_qspi_l1_isr()  local
1322 	struct bcm_qspi_soc_intc *soc_intc = qspi->soc_intc;  in bcm_qspi_l1_isr()
1326 		u32 status = soc_intc->bcm_qspi_get_int_status(soc_intc);  in bcm_qspi_l1_isr()
1387 static void bcm_qspi_bspi_init(struct bcm_qspi *qspi)  in bcm_qspi_bspi_init()  argument
1391 	val = bcm_qspi_read(qspi, BSPI, BSPI_REVISION_ID);  in bcm_qspi_bspi_init()
1392 	qspi->bspi_maj_rev = (val >> 8) & 0xff;  in bcm_qspi_bspi_init()
1393 	qspi->bspi_min_rev = val & 0xff;  in bcm_qspi_bspi_init()
1394 	if (!(bcm_qspi_bspi_ver_three(qspi))) {  in bcm_qspi_bspi_init()
1395 		/* Force mapping of BSPI address -> flash offset */  in bcm_qspi_bspi_init()
1396 		bcm_qspi_write(qspi, BSPI, BSPI_BSPI_XOR_VALUE, 0);  in bcm_qspi_bspi_init()
1397 		bcm_qspi_write(qspi, BSPI, BSPI_BSPI_XOR_ENABLE, 1);  in bcm_qspi_bspi_init()
1399 	qspi->bspi_enabled = 1;  in bcm_qspi_bspi_init()
1400 	bcm_qspi_disable_bspi(qspi);  in bcm_qspi_bspi_init()
1401 	bcm_qspi_write(qspi, BSPI, BSPI_B0_CTRL, 0);  in bcm_qspi_bspi_init()
1402 	bcm_qspi_write(qspi, BSPI, BSPI_B1_CTRL, 0);  in bcm_qspi_bspi_init()
1405 static void bcm_qspi_hw_init(struct bcm_qspi *qspi)  in bcm_qspi_hw_init()  argument
1409 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR1_LSB, 0);  in bcm_qspi_hw_init()
1410 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR1_MSB, 0);  in bcm_qspi_hw_init()
1411 	bcm_qspi_write(qspi, MSPI, MSPI_NEWQP, 0);  in bcm_qspi_hw_init()
1412 	bcm_qspi_write(qspi, MSPI, MSPI_ENDQP, 0);  in bcm_qspi_hw_init()
1413 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0x20);  in bcm_qspi_hw_init()
1417 	parms.speed_hz = qspi->max_speed_hz;  in bcm_qspi_hw_init()
1418 	bcm_qspi_hw_set_parms(qspi, &parms);  in bcm_qspi_hw_init()
1420 	if (has_bspi(qspi))  in bcm_qspi_hw_init()
1421 		bcm_qspi_bspi_init(qspi);  in bcm_qspi_hw_init()
1424 static void bcm_qspi_hw_uninit(struct bcm_qspi *qspi)  in bcm_qspi_hw_uninit()  argument
1426 	u32 status = bcm_qspi_read(qspi, MSPI, MSPI_MSPI_STATUS);  in bcm_qspi_hw_uninit()
1428 	bcm_qspi_write(qspi, MSPI, MSPI_SPCR2, 0);  in bcm_qspi_hw_uninit()
1429 	if (has_bspi(qspi))  in bcm_qspi_hw_uninit()
1430 		bcm_qspi_write(qspi, MSPI, MSPI_WRITE_LOCK, 0);  in bcm_qspi_hw_uninit()
1433 	bcm_qspi_write(qspi, MSPI, MSPI_MSPI_STATUS, status & ~1);  in bcm_qspi_hw_uninit()
1462 		.compatible = "brcm,spi-bcm7445-qspi",
1467 		.compatible = "brcm,spi-bcm-qspi",
1471 		.compatible = "brcm,spi-bcm7216-qspi",
1475 		.compatible = "brcm,spi-bcm7278-qspi",
1487 	struct device *dev = &pdev->dev;  in bcm_qspi_probe()
1488 	struct bcm_qspi *qspi;  in bcm_qspi_probe()  local
1497 	/* We only support device-tree instantiation */  in bcm_qspi_probe()
1498 	if (!dev->of_node)  in bcm_qspi_probe()
1499 		return -ENODEV;  in bcm_qspi_probe()
1501 	of_id = of_match_node(bcm_qspi_of_match, dev->of_node);  in bcm_qspi_probe()
1503 		return -ENODEV;  in bcm_qspi_probe()
1505 	data = of_id->data;  in bcm_qspi_probe()
1510 		return -ENOMEM;  in bcm_qspi_probe()
1513 	qspi = spi_controller_get_devdata(host);  in bcm_qspi_probe()
1515 	qspi->clk = devm_clk_get_optional(&pdev->dev, NULL);  in bcm_qspi_probe()
1516 	if (IS_ERR(qspi->clk))  in bcm_qspi_probe()
1517 		return PTR_ERR(qspi->clk);  in bcm_qspi_probe()
1519 	qspi->pdev = pdev;  in bcm_qspi_probe()
1520 	qspi->trans_pos.trans = NULL;  in bcm_qspi_probe()
1521 	qspi->trans_pos.byte = 0;  in bcm_qspi_probe()
1522 	qspi->trans_pos.mspi_last_trans = true;  in bcm_qspi_probe()
1523 	qspi->host = host;  in bcm_qspi_probe()
1525 	host->bus_num = -1;  in bcm_qspi_probe()
1526 	host->mode_bits = SPI_CPHA | SPI_CPOL | SPI_RX_DUAL | SPI_RX_QUAD |  in bcm_qspi_probe()
1528 	host->setup = bcm_qspi_setup;  in bcm_qspi_probe()
1529 	host->transfer_one = bcm_qspi_transfer_one;  in bcm_qspi_probe()
1530 	host->mem_ops = &bcm_qspi_mem_ops;  in bcm_qspi_probe()
1531 	host->cleanup = bcm_qspi_cleanup;  in bcm_qspi_probe()
1532 	host->dev.of_node = dev->of_node;  in bcm_qspi_probe()
1533 	host->num_chipselect = NUM_CHIPSELECT;  in bcm_qspi_probe()
1534 	host->use_gpio_descriptors = true;  in bcm_qspi_probe()
1536 	qspi->big_endian = of_device_is_big_endian(dev->of_node);  in bcm_qspi_probe()
1538 	if (!of_property_read_u32(dev->of_node, "num-cs", &val))  in bcm_qspi_probe()
1539 		host->num_chipselect = val;  in bcm_qspi_probe()
1546 	qspi->base[MSPI]  = devm_ioremap_resource(dev, res);  in bcm_qspi_probe()
1547 	if (IS_ERR(qspi->base[MSPI]))  in bcm_qspi_probe()
1548 		return PTR_ERR(qspi->base[MSPI]);  in bcm_qspi_probe()
1552 		qspi->base[BSPI]  = devm_ioremap_resource(dev, res);  in bcm_qspi_probe()
1553 		if (IS_ERR(qspi->base[BSPI]))  in bcm_qspi_probe()
1554 			return PTR_ERR(qspi->base[BSPI]);  in bcm_qspi_probe()
1555 		qspi->bspi_mode = true;  in bcm_qspi_probe()
1557 		qspi->bspi_mode = false;  in bcm_qspi_probe()
1560 	dev_info(dev, "using %smspi mode\n", qspi->bspi_mode ? "bspi-" : "");  in bcm_qspi_probe()
1564 		qspi->base[CHIP_SELECT]  = devm_ioremap_resource(dev, res);  in bcm_qspi_probe()
1565 		if (IS_ERR(qspi->base[CHIP_SELECT]))  in bcm_qspi_probe()
1566 			return PTR_ERR(qspi->base[CHIP_SELECT]);  in bcm_qspi_probe()
1569 	qspi->dev_ids = kcalloc(num_irqs, sizeof(struct bcm_qspi_dev_id),  in bcm_qspi_probe()
1571 	if (!qspi->dev_ids)  in bcm_qspi_probe()
1572 		return -ENOMEM;  in bcm_qspi_probe()
1575 	 * Some SoCs integrate spi controller (e.g., its interrupt bits)  in bcm_qspi_probe()
1579 		qspi->soc_intc = soc_intc;  in bcm_qspi_probe()
1580 		soc_intc->bcm_qspi_int_set(soc_intc, MSPI_DONE, true);  in bcm_qspi_probe()
1582 		qspi->soc_intc = NULL;  in bcm_qspi_probe()
1585 	if (qspi->clk) {  in bcm_qspi_probe()
1586 		ret = clk_prepare_enable(qspi->clk);  in bcm_qspi_probe()
1591 		qspi->base_clk = clk_get_rate(qspi->clk);  in bcm_qspi_probe()
1593 		qspi->base_clk = MSPI_BASE_FREQ;  in bcm_qspi_probe()
1596 	if (data->has_mspi_rev) {  in bcm_qspi_probe()
1597 		rev = bcm_qspi_read(qspi, MSPI, MSPI_REV);  in bcm_qspi_probe()
1603 	qspi->mspi_maj_rev = (rev >> 4) & 0xf;  in bcm_qspi_probe()
1604 	qspi->mspi_min_rev = rev & 0xf;  in bcm_qspi_probe()
1605 	qspi->mspi_spcr3_sysclk = data->has_spcr3_sysclk;  in bcm_qspi_probe()
1607 	qspi->max_speed_hz = qspi->base_clk / (bcm_qspi_spbr_min(qspi) * 2);  in bcm_qspi_probe()
1613 	bcm_qspi_hw_uninit(qspi);  in bcm_qspi_probe()
1616 		irq = -1;  in bcm_qspi_probe()
1627 			ret = devm_request_irq(&pdev->dev, irq,  in bcm_qspi_probe()
1630 					       &qspi->dev_ids[val]);  in bcm_qspi_probe()
1632 				dev_err(&pdev->dev, "IRQ %s not found\n", name);  in bcm_qspi_probe()
1636 			qspi->dev_ids[val].dev = qspi;  in bcm_qspi_probe()
1637 			qspi->dev_ids[val].irqp = &qspi_irq_tab[val];  in bcm_qspi_probe()
1639 			dev_dbg(&pdev->dev, "registered IRQ %s %d\n",  in bcm_qspi_probe()
1646 		dev_err(&pdev->dev, "no IRQs registered, cannot init driver\n");  in bcm_qspi_probe()
1647 		ret = -EINVAL;  in bcm_qspi_probe()
1651 	bcm_qspi_hw_init(qspi);  in bcm_qspi_probe()
1652 	init_completion(&qspi->mspi_done);  in bcm_qspi_probe()
1653 	init_completion(&qspi->bspi_done);  in bcm_qspi_probe()
1654 	qspi->curr_cs = -1;  in bcm_qspi_probe()
1656 	platform_set_drvdata(pdev, qspi);  in bcm_qspi_probe()
1658 	qspi->xfer_mode.width = -1;  in bcm_qspi_probe()
1659 	qspi->xfer_mode.addrlen = -1;  in bcm_qspi_probe()
1660 	qspi->xfer_mode.hp = -1;  in bcm_qspi_probe()
1671 	bcm_qspi_hw_uninit(qspi);  in bcm_qspi_probe()
1673 	clk_disable_unprepare(qspi->clk);  in bcm_qspi_probe()
1675 	kfree(qspi->dev_ids);  in bcm_qspi_probe()
1683 	struct bcm_qspi *qspi = platform_get_drvdata(pdev);  in bcm_qspi_remove()  local
1685 	spi_unregister_controller(qspi->host);  in bcm_qspi_remove()
1686 	bcm_qspi_hw_uninit(qspi);  in bcm_qspi_remove()
1687 	clk_disable_unprepare(qspi->clk);  in bcm_qspi_remove()
1688 	kfree(qspi->dev_ids);  in bcm_qspi_remove()
1696 	struct bcm_qspi *qspi = dev_get_drvdata(dev);  in bcm_qspi_suspend()  local
1699 	if (!bcm_qspi_bspi_ver_three(qspi))  in bcm_qspi_suspend()
1700 		qspi->s3_strap_override_ctrl =  in bcm_qspi_suspend()
1701 			bcm_qspi_read(qspi, BSPI, BSPI_STRAP_OVERRIDE_CTRL);  in bcm_qspi_suspend()
1703 	spi_controller_suspend(qspi->host);  in bcm_qspi_suspend()
1704 	clk_disable_unprepare(qspi->clk);  in bcm_qspi_suspend()
1705 	bcm_qspi_hw_uninit(qspi);  in bcm_qspi_suspend()
1712 	struct bcm_qspi *qspi = dev_get_drvdata(dev);  in bcm_qspi_resume()  local
1715 	bcm_qspi_hw_init(qspi);  in bcm_qspi_resume()
1716 	bcm_qspi_chip_select(qspi, qspi->curr_cs);  in bcm_qspi_resume()
1717 	if (qspi->soc_intc)  in bcm_qspi_resume()
1719 		qspi->soc_intc->bcm_qspi_int_set(qspi->soc_intc, MSPI_DONE,  in bcm_qspi_resume()
1722 	ret = clk_prepare_enable(qspi->clk);  in bcm_qspi_resume()
1724 		spi_controller_resume(qspi->host);  in bcm_qspi_resume()
1735 MODULE_DESCRIPTION("Broadcom QSPI driver");