1 // SPDX-License-Identifier: GPL-2.0-only
4  * 8250-core based driver for Broadcom ns16550a UARTs
20 #include <linux/dma-mapping.h>
28 /* Register definitions for UART DMA block. Version 1.1 or later. */
157 #define UDMA_RX_BUFS_REG_OFFSET (UDMA_RX_BUF1_PTR_LO - UDMA_RX_BUF0_PTR_LO)
169 #define UDMA_TX_BUFS_REG_OFFSET (UDMA_TX_BUF1_PTR_LO - UDMA_TX_BUF0_PTR_LO)
251 	return readl(priv->regs[reg_type] + offset);  in udma_readl()
257 	writel(value, priv->regs[reg_type] + offset);  in udma_writel()
263 	void __iomem *reg = priv->regs[reg_type] + offset;  in udma_set()
274 	void __iomem *reg = priv->regs[reg_type] + offset;  in udma_unset()
283  * The UART DMA engine hardware can be used by multiple UARTS, but
285  * the first UART to request the DMA engine will get it and any
305 			if (waits-- == 0)  in brcmuart_arbitration()
339 	daddr = priv->rx_addr;  in brcmuart_init_dma_hardware()
352 	daddr = priv->tx_addr;  in brcmuart_init_dma_hardware()
369 	struct brcmuart_priv *priv = p->port.private_data;  in start_rx_dma()
382 	priv->rx_next_buf = 0;  in start_rx_dma()
385 	priv->rx_running = true;  in start_rx_dma()
390 	struct brcmuart_priv *priv = p->port.private_data;  in stop_rx_dma()
393 	if (priv->rx_running)  in stop_rx_dma()
399 	struct brcmuart_priv *priv = p->port.private_data;  in stop_tx_dma()
406 	priv->tx_running = false;  in stop_tx_dma()
416 	struct brcmuart_priv *priv = p->port.private_data;  in brcmuart_tx_dma()
417 	struct circ_buf *xmit = &p->port.state->xmit;  in brcmuart_tx_dma()
420 	if (uart_tx_stopped(&p->port) || priv->tx_running ||  in brcmuart_tx_dma()
424 	tx_size = CIRC_CNT_TO_END(xmit->head, xmit->tail, UART_XMIT_SIZE);  in brcmuart_tx_dma()
426 	priv->dma.tx_err = 0;  in brcmuart_tx_dma()
427 	memcpy(priv->tx_buf, &xmit->buf[xmit->tail], tx_size);  in brcmuart_tx_dma()
428 	xmit->tail += tx_size;  in brcmuart_tx_dma()
429 	xmit->tail &= UART_XMIT_SIZE - 1;  in brcmuart_tx_dma()
430 	p->port.icount.tx += tx_size;  in brcmuart_tx_dma()
433 		uart_write_wakeup(&p->port);  in brcmuart_tx_dma()
439 	priv->tx_running = true;  in brcmuart_tx_dma()
446 	struct brcmuart_priv *priv = up->private_data;  in brcmuart_rx_buf_done_isr()
447 	struct tty_port *tty_port = &up->state->port;  in brcmuart_rx_buf_done_isr()
457 		dev_err(up->dev, "RX done interrupt but DATA_RDY not found\n");  in brcmuart_rx_buf_done_isr()
464 			up->icount.overrun++;  in brcmuart_rx_buf_done_isr()
465 			dev_warn(up->dev, "RX OVERRUN Error\n");  in brcmuart_rx_buf_done_isr()
468 			up->icount.frame++;  in brcmuart_rx_buf_done_isr()
469 			dev_warn(up->dev, "RX FRAMING Error\n");  in brcmuart_rx_buf_done_isr()
472 			up->icount.parity++;  in brcmuart_rx_buf_done_isr()
473 			dev_warn(up->dev, "RX PARITY Error\n");  in brcmuart_rx_buf_done_isr()
478 		priv->rx_bufs + (index * RX_BUF_SIZE),  in brcmuart_rx_buf_done_isr()
481 		dev_warn(up->dev, "Flip buffer overrun of %d bytes\n",  in brcmuart_rx_buf_done_isr()
482 			 length - copied);  in brcmuart_rx_buf_done_isr()
483 		up->icount.overrun += length - copied;  in brcmuart_rx_buf_done_isr()
485 	up->icount.rx += length;  in brcmuart_rx_buf_done_isr()
487 		priv->dma_rx_partial_buf++;  in brcmuart_rx_buf_done_isr()
493 		priv->rx_missing_close_timeout++;  in brcmuart_rx_buf_done_isr()
495 		priv->dma_rx_full_buf++;  in brcmuart_rx_buf_done_isr()
502 	struct brcmuart_priv *priv = up->private_data;  in brcmuart_rx_isr()
503 	struct device *dev = up->dev;  in brcmuart_rx_isr()
509 		check_isr = UDMA_INTR_RX_READY_BUF0 << priv->rx_next_buf;  in brcmuart_rx_isr()
511 			brcmuart_rx_buf_done_isr(up, priv->rx_next_buf);  in brcmuart_rx_isr()
520 				priv->rx_err++;  in brcmuart_rx_isr()
522 				priv->rx_timeout++;  in brcmuart_rx_isr()
526 				priv->rx_abort++;  in brcmuart_rx_isr()
527 			priv->rx_running = false;  in brcmuart_rx_isr()
529 		/* If not ABORT, re-enable RX buffer */  in brcmuart_rx_isr()
532 				   UDMA_RX_BUFx_STATUS(priv->rx_next_buf),  in brcmuart_rx_isr()
535 		priv->rx_next_buf++;  in brcmuart_rx_isr()
536 		if (priv->rx_next_buf == RX_BUFS_COUNT)  in brcmuart_rx_isr()
537 			priv->rx_next_buf = 0;  in brcmuart_rx_isr()
543 	struct brcmuart_priv *priv = up->private_data;  in brcmuart_tx_isr()
544 	struct device *dev = up->dev;  in brcmuart_tx_isr()
546 	struct circ_buf	*xmit = &port_8250->port.state->xmit;  in brcmuart_tx_isr()
549 		if (priv->tx_running)  in brcmuart_tx_isr()
553 	priv->tx_running = false;  in brcmuart_tx_isr()
561 	struct device *dev = up->dev;  in brcmuart_isr()
562 	struct brcmuart_priv *priv = up->private_data;  in brcmuart_isr()
572 	spin_lock_irqsave(&up->lock, flags);  in brcmuart_isr()
586 	spin_unlock_irqrestore(&up->lock, flags);  in brcmuart_isr()
594 	struct brcmuart_priv *priv = up->port.private_data;  in brcmuart_startup()
596 	priv->shutdown = false;  in brcmuart_startup()
599 	 * prevent serial8250_do_startup() from allocating non-existent  in brcmuart_startup()
602 	up->dma = NULL;  in brcmuart_startup()
605 	if (!priv->dma_enabled)  in brcmuart_startup()
611 	up->ier &= ~UART_IER_RDI;  in brcmuart_startup()
612 	serial_port_out(port, UART_IER, up->ier);  in brcmuart_startup()
614 	priv->tx_running = false;  in brcmuart_startup()
615 	priv->dma.rx_dma = NULL;  in brcmuart_startup()
616 	priv->dma.tx_dma = brcmuart_tx_dma;  in brcmuart_startup()
617 	up->dma = &priv->dma;  in brcmuart_startup()
627 	struct brcmuart_priv *priv = up->port.private_data;  in brcmuart_shutdown()
630 	spin_lock_irqsave(&port->lock, flags);  in brcmuart_shutdown()
631 	priv->shutdown = true;  in brcmuart_shutdown()
632 	if (priv->dma_enabled) {  in brcmuart_shutdown()
644 	up->dma = NULL;  in brcmuart_shutdown()
646 	spin_unlock_irqrestore(&port->lock, flags);  in brcmuart_shutdown()
659 	priv->default_mux_rate = clk_get_rate(priv->baud_mux_clk);  in init_real_clk_rates()
660 	for (x = 0; x < ARRAY_SIZE(priv->real_rates); x++) {  in init_real_clk_rates()
661 		if (priv->rate_table[x] == 0) {  in init_real_clk_rates()
662 			priv->real_rates[x] = 0;  in init_real_clk_rates()
665 		rc = clk_set_rate(priv->baud_mux_clk, priv->rate_table[x]);  in init_real_clk_rates()
668 				priv->rate_table[x]);  in init_real_clk_rates()
669 			priv->real_rates[x] = priv->rate_table[x];  in init_real_clk_rates()
671 			priv->real_rates[x] = clk_get_rate(priv->baud_mux_clk);  in init_real_clk_rates()
674 	clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);  in init_real_clk_rates()
685 	int best_index = -1;  in set_clock_mux()
694 	if (priv->baud_mux_clk == NULL)  in set_clock_mux()
698 	for (i = 0; i < ARRAY_SIZE(priv->real_rates); i++) {  in set_clock_mux()
699 		if (priv->real_rates[i] == 0)  in set_clock_mux()
701 		rate = priv->real_rates[i] / 16;  in set_clock_mux()
714 			hires_err = (hires_err - hires_baud);  in set_clock_mux()
716 			hires_err = (hires_baud - hires_err);  in set_clock_mux()
719 		dev_dbg(up->dev,  in set_clock_mux()
721 			baud, priv->real_rates[i], percent / 100,  in set_clock_mux()
729 	if (best_index == -1) {  in set_clock_mux()
730 		dev_err(up->dev, "Error, %d BAUD rate is too fast.\n", baud);  in set_clock_mux()
733 	rate = priv->real_rates[best_index];  in set_clock_mux()
734 	rc = clk_set_rate(priv->baud_mux_clk, rate);  in set_clock_mux()
736 		dev_err(up->dev, "Error selecting BAUD MUX clock\n");  in set_clock_mux()
740 		dev_err(up->dev, "Error, baud: %d has %u.%u%% error\n",  in set_clock_mux()
744 	dev_dbg(up->dev, "Selecting BAUD MUX rate: %u\n", rate);  in set_clock_mux()
745 	dev_dbg(up->dev, "Requested baud: %u, Actual baud: %u\n",  in set_clock_mux()
751 	priv->char_wait = ns_to_ktime(i);  in set_clock_mux()
753 	up->uartclk = rate;  in set_clock_mux()
761 	struct brcmuart_priv *priv = up->private_data;  in brcmstb_set_termios()
763 	if (priv->dma_enabled)  in brcmstb_set_termios()
767 	if (p8250->mcr & UART_MCR_AFE)  in brcmstb_set_termios()
768 		p8250->port.status |= UPSTAT_AUTOCTS;  in brcmstb_set_termios()
769 	if (priv->dma_enabled)  in brcmstb_set_termios()
776 	struct brcmuart_priv *priv = p->private_data;  in brcmuart_handle_irq()
788 	if (((iir & UART_IIR_ID) == UART_IIR_RX_TIMEOUT) && !(priv->shutdown)) {  in brcmuart_handle_irq()
789 		spin_lock_irqsave(&p->lock, flags);  in brcmuart_handle_irq()
800 			if ((ier & UART_IER_RDI) && (up->mcr & UART_MCR_AFE)) {  in brcmuart_handle_irq()
806 				hrtimer_start(&priv->hrt, priv->char_wait,  in brcmuart_handle_irq()
814 		spin_unlock_irqrestore(&p->lock, flags);  in brcmuart_handle_irq()
824 	struct uart_port *p = priv->up;  in brcmuart_hrtimer_func()
829 	if (priv->shutdown)  in brcmuart_hrtimer_func()
832 	spin_lock_irqsave(&p->lock, flags);  in brcmuart_hrtimer_func()
841 		priv->rx_bad_timeout_no_char++;  in brcmuart_hrtimer_func()
843 		priv->rx_bad_timeout_late_char++;  in brcmuart_hrtimer_func()
846 	/* re-enable receive unless upper layer has disabled it */  in brcmuart_hrtimer_func()
847 	if ((up->ier & (UART_IER_RLSI | UART_IER_RDI)) ==  in brcmuart_hrtimer_func()
856 	spin_unlock_irqrestore(&p->lock, flags);  in brcmuart_hrtimer_func()
862 		.compatible = "brcm,bcm7278-uart",
866 		.compatible = "brcm,bcm7271-uart",
876 	if (priv->rx_bufs)  in brcmuart_free_bufs()
877 		dma_free_coherent(dev, priv->rx_size, priv->rx_bufs,  in brcmuart_free_bufs()
878 				  priv->rx_addr);  in brcmuart_free_bufs()
879 	if (priv->tx_buf)  in brcmuart_free_bufs()
880 		dma_free_coherent(dev, priv->tx_size, priv->tx_buf,  in brcmuart_free_bufs()
881 				  priv->tx_addr);  in brcmuart_free_bufs()
886 	struct brcmuart_priv *priv = port->private_data;  in brcmuart_throttle()
893 	struct brcmuart_priv *priv = port->private_data;  in brcmuart_unthrottle()
901 	struct brcmuart_priv *priv = s->private;  in debugfs_stats_show()
904 		   priv->rx_err);  in debugfs_stats_show()
906 		   priv->rx_timeout);  in debugfs_stats_show()
908 		   priv->rx_abort);  in debugfs_stats_show()
910 		   priv->rx_bad_timeout_late_char);  in debugfs_stats_show()
912 		   priv->rx_bad_timeout_no_char);  in debugfs_stats_show()
914 		   priv->rx_missing_close_timeout);  in debugfs_stats_show()
915 	if (priv->dma_enabled) {  in debugfs_stats_show()
917 			   priv->dma_rx_partial_buf);  in debugfs_stats_show()
919 			   priv->dma_rx_full_buf);  in debugfs_stats_show()
928 	priv->debugfs_dir = debugfs_create_dir(device, brcmuart_debugfs_root);  in brcmuart_init_debugfs()
929 	debugfs_create_file("stats", 0444, priv->debugfs_dir, priv,  in brcmuart_init_debugfs()
937 	struct device_node *np = pdev->dev.of_node;  in brcmuart_probe()
940 	struct device *dev = &pdev->dev;  in brcmuart_probe()
952 		"uart", "dma_rx", "dma_tx", "dma_intr2", "dma_arb"  in brcmuart_probe()
961 		return -ENOMEM;  in brcmuart_probe()
964 	if (!of_id || !of_id->data)  in brcmuart_probe()
965 		priv->rate_table = brcmstb_rate_table;  in brcmuart_probe()
967 		priv->rate_table = of_id->data;  in brcmuart_probe()
974 		priv->regs[x] =	devm_ioremap(dev, regs->start,  in brcmuart_probe()
976 		if (!priv->regs[x])  in brcmuart_probe()
977 			return -ENOMEM;  in brcmuart_probe()
979 			mapbase = regs->start;  in brcmuart_probe()
980 			membase = priv->regs[x];  in brcmuart_probe()
984 	/* We should have just the uart base registers or all the registers */  in brcmuart_probe()
987 		return -EINVAL;  in brcmuart_probe()
1002 				priv->dma_enabled = true;  in brcmuart_probe()
1010 				"Timeout arbitrating for UART DMA hardware\n");  in brcmuart_probe()
1014 	of_property_read_u32(np, "clock-frequency", &clk_rate);  in brcmuart_probe()
1019 		if (PTR_ERR(baud_mux_clk) == -EPROBE_DEFER)  in brcmuart_probe()
1020 			return -EPROBE_DEFER;  in brcmuart_probe()
1027 		priv->baud_mux_clk = baud_mux_clk;  in brcmuart_probe()
1029 		clk_rate = priv->default_mux_rate;  in brcmuart_probe()
1033 		dev_err(dev, "clock-frequency or clk not defined\n");  in brcmuart_probe()
1034 		return -EINVAL;  in brcmuart_probe()
1037 	dev_dbg(dev, "DMA is %senabled\n", priv->dma_enabled ? "" : "not ");  in brcmuart_probe()
1063 	hrtimer_init(&priv->hrt, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);  in brcmuart_probe()
1064 	priv->hrt.function = brcmuart_hrtimer_func;  in brcmuart_probe()
1072 	if (priv->dma_enabled) {  in brcmuart_probe()
1073 		priv->rx_size = RX_BUF_SIZE * RX_BUFS_COUNT;  in brcmuart_probe()
1074 		priv->rx_bufs = dma_alloc_coherent(dev,  in brcmuart_probe()
1075 						   priv->rx_size,  in brcmuart_probe()
1076 						   &priv->rx_addr, GFP_KERNEL);  in brcmuart_probe()
1077 		if (!priv->rx_bufs) {  in brcmuart_probe()
1078 			ret = -ENOMEM;  in brcmuart_probe()
1081 		priv->tx_size = UART_XMIT_SIZE;  in brcmuart_probe()
1082 		priv->tx_buf = dma_alloc_coherent(dev,  in brcmuart_probe()
1083 						  priv->tx_size,  in brcmuart_probe()
1084 						  &priv->tx_addr, GFP_KERNEL);  in brcmuart_probe()
1085 		if (!priv->tx_buf) {  in brcmuart_probe()
1086 			ret = -ENOMEM;  in brcmuart_probe()
1096 	priv->line = ret;  in brcmuart_probe()
1098 	priv->up = &new_port->port;  in brcmuart_probe()
1099 	if (priv->dma_enabled) {  in brcmuart_probe()
1107 				IRQF_SHARED, "uart DMA irq", &new_port->port);  in brcmuart_probe()
1114 	brcmuart_init_debugfs(priv, dev_name(&pdev->dev));  in brcmuart_probe()
1118 	serial8250_unregister_port(priv->line);  in brcmuart_probe()
1129 	debugfs_remove_recursive(priv->debugfs_dir);  in brcmuart_remove()
1130 	hrtimer_cancel(&priv->hrt);  in brcmuart_remove()
1131 	serial8250_unregister_port(priv->line);  in brcmuart_remove()
1132 	brcmuart_free_bufs(&pdev->dev, priv);  in brcmuart_remove()
1140 	struct uart_8250_port *up = serial8250_get_port(priv->line);  in brcmuart_suspend()
1141 	struct uart_port *port = &up->port;  in brcmuart_suspend()
1148 	spin_lock_irqsave(&port->lock, flags);  in brcmuart_suspend()
1149 	priv->saved_mctrl = port->mctrl;  in brcmuart_suspend()
1150 	port->mctrl &= ~TIOCM_RTS;  in brcmuart_suspend()
1151 	spin_unlock_irqrestore(&port->lock, flags);  in brcmuart_suspend()
1153 	serial8250_suspend_port(priv->line);  in brcmuart_suspend()
1154 	clk_disable_unprepare(priv->baud_mux_clk);  in brcmuart_suspend()
1162 	struct uart_8250_port *up = serial8250_get_port(priv->line);  in brcmuart_resume()
1163 	struct uart_port *port = &up->port;  in brcmuart_resume()
1167 	ret = clk_prepare_enable(priv->baud_mux_clk);  in brcmuart_resume()
1175 	ret = clk_set_rate(priv->baud_mux_clk, priv->default_mux_rate);  in brcmuart_resume()
1178 	if (priv->dma_enabled) {  in brcmuart_resume()
1181 			return(-EBUSY);  in brcmuart_resume()
1184 		start_rx_dma(serial8250_get_port(priv->line));  in brcmuart_resume()
1186 	serial8250_resume_port(priv->line);  in brcmuart_resume()
1188 	if (priv->saved_mctrl & TIOCM_RTS) {  in brcmuart_resume()
1190 		spin_lock_irqsave(&port->lock, flags);  in brcmuart_resume()
1191 		port->mctrl |= TIOCM_RTS;  in brcmuart_resume()
1192 		port->ops->set_mctrl(port, port->mctrl);  in brcmuart_resume()
1193 		spin_unlock_irqrestore(&port->lock, flags);  in brcmuart_resume()
1205 		.name	= "bcm7271-uart",