122 	struct uart_port port;  member
134 static int sprd_start_dma_rx(struct uart_port *port);
135 static int sprd_tx_dma_config(struct uart_port *port);
137 static inline unsigned int serial_in(struct uart_port *port,  in serial_in()  argument
140 	return readl_relaxed(port->membase + offset);  in serial_in()
143 static inline void serial_out(struct uart_port *port, unsigned int offset,  in serial_out()  argument
146 	writel_relaxed(value, port->membase + offset);  in serial_out()
149 static unsigned int sprd_tx_empty(struct uart_port *port)  in sprd_tx_empty()  argument
151 	if (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_tx_empty()
157 static unsigned int sprd_get_mctrl(struct uart_port *port)  in sprd_get_mctrl()  argument
162 static void sprd_set_mctrl(struct uart_port *port, unsigned int mctrl)  in sprd_set_mctrl()  argument
164 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_set_mctrl()
171 	serial_out(port, SPRD_CTL1, val);  in sprd_set_mctrl()
174 static void sprd_stop_rx(struct uart_port *port)  in sprd_stop_rx()  argument
177 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_rx()
183 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_rx()
184 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_rx()
189 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_rx()
190 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_rx()
193 static void sprd_uart_dma_enable(struct uart_port *port, bool enable)  in sprd_uart_dma_enable()  argument
195 	u32 val = serial_in(port, SPRD_CTL1);  in sprd_uart_dma_enable()
202 	serial_out(port, SPRD_CTL1, val);  in sprd_uart_dma_enable()
205 static void sprd_stop_tx_dma(struct uart_port *port)  in sprd_stop_tx_dma()  argument
208 		container_of(port, struct sprd_uart_port, port);  in sprd_stop_tx_dma()
209 	struct circ_buf *xmit = &port->state->xmit;  in sprd_stop_tx_dma()
219 		port->icount.tx += trans_len;  in sprd_stop_tx_dma()
220 		dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_stop_tx_dma()
228 static int sprd_tx_buf_remap(struct uart_port *port)  in sprd_tx_buf_remap()  argument
231 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_buf_remap()
232 	struct circ_buf *xmit = &port->state->xmit;  in sprd_tx_buf_remap()
237 	sp->tx_dma.phys_addr = dma_map_single(port->dev,  in sprd_tx_buf_remap()
241 	return dma_mapping_error(port->dev, sp->tx_dma.phys_addr);  in sprd_tx_buf_remap()
246 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_tx_dma()  local
248 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_tx_dma()
249 	struct circ_buf *xmit = &port->state->xmit;  in sprd_complete_tx_dma()
252 	spin_lock_irqsave(&port->lock, flags);  in sprd_complete_tx_dma()
253 	dma_unmap_single(port->dev, sp->tx_dma.phys_addr,  in sprd_complete_tx_dma()
257 	port->icount.tx += sp->tx_dma.trans_len;  in sprd_complete_tx_dma()
260 		uart_write_wakeup(port);  in sprd_complete_tx_dma()
262 	if (uart_circ_empty(xmit) || sprd_tx_buf_remap(port) ||  in sprd_complete_tx_dma()
263 	    sprd_tx_dma_config(port))  in sprd_complete_tx_dma()
266 	spin_unlock_irqrestore(&port->lock, flags);  in sprd_complete_tx_dma()
269 static int sprd_uart_dma_submit(struct uart_port *port,  in sprd_uart_dma_submit()  argument
288 	dma_des->callback_param = port;  in sprd_uart_dma_submit()
299 static int sprd_tx_dma_config(struct uart_port *port)  in sprd_tx_dma_config()  argument
302 		container_of(port, struct sprd_uart_port, port);  in sprd_tx_dma_config()
307 		.dst_addr = port->mapbase + SPRD_TXD,  in sprd_tx_dma_config()
317 	return sprd_uart_dma_submit(port, &sp->tx_dma, sp->tx_dma.trans_len,  in sprd_tx_dma_config()
321 static void sprd_start_tx_dma(struct uart_port *port)  in sprd_start_tx_dma()  argument
324 		container_of(port, struct sprd_uart_port, port);  in sprd_start_tx_dma()
325 	struct circ_buf *xmit = &port->state->xmit;  in sprd_start_tx_dma()
327 	if (port->x_char) {  in sprd_start_tx_dma()
328 		serial_out(port, SPRD_TXD, port->x_char);  in sprd_start_tx_dma()
329 		port->icount.tx++;  in sprd_start_tx_dma()
330 		port->x_char = 0;  in sprd_start_tx_dma()
334 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {  in sprd_start_tx_dma()
335 		sprd_stop_tx_dma(port);  in sprd_start_tx_dma()
342 	if (sprd_tx_buf_remap(port) || sprd_tx_dma_config(port))  in sprd_start_tx_dma()
346 static void sprd_rx_full_thld(struct uart_port *port, u32 thld)  in sprd_rx_full_thld()  argument
348 	u32 val = serial_in(port, SPRD_CTL2);  in sprd_rx_full_thld()
352 	serial_out(port, SPRD_CTL2, val);  in sprd_rx_full_thld()
357 	sp->rx_dma.virt = dma_alloc_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_alloc_buf()
368 		dma_free_coherent(sp->port.dev, SPRD_UART_RX_SIZE,  in sprd_rx_free_buf()
373 static int sprd_rx_dma_config(struct uart_port *port, u32 burst)  in sprd_rx_dma_config()  argument
376 		container_of(port, struct sprd_uart_port, port);  in sprd_rx_dma_config()
378 		.src_addr = port->mapbase + SPRD_RXD,  in sprd_rx_dma_config()
387 static void sprd_uart_dma_rx(struct uart_port *port)  in sprd_uart_dma_rx()  argument
390 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_rx()
391 	struct tty_port *tty = &port->state->port;  in sprd_uart_dma_rx()
393 	port->icount.rx += sp->rx_dma.trans_len;  in sprd_uart_dma_rx()
398 static void sprd_uart_dma_irq(struct uart_port *port)  in sprd_uart_dma_irq()  argument
401 		container_of(port, struct sprd_uart_port, port);  in sprd_uart_dma_irq()
408 		sprd_stop_rx(port);  in sprd_uart_dma_irq()
422 	sprd_uart_dma_rx(port);  in sprd_uart_dma_irq()
428 	struct uart_port *port = (struct uart_port *)data;  in sprd_complete_rx_dma()  local
430 		container_of(port, struct sprd_uart_port, port);  in sprd_complete_rx_dma()
435 	spin_lock_irqsave(&port->lock, flags);  in sprd_complete_rx_dma()
440 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
441 		spin_unlock_irqrestore(&port->lock, flags);  in sprd_complete_rx_dma()
448 		sprd_uart_dma_rx(port);  in sprd_complete_rx_dma()
452 	if (sprd_start_dma_rx(port))  in sprd_complete_rx_dma()
453 		sprd_stop_rx(port);  in sprd_complete_rx_dma()
455 	spin_unlock_irqrestore(&port->lock, flags);  in sprd_complete_rx_dma()
458 static int sprd_start_dma_rx(struct uart_port *port)  in sprd_start_dma_rx()  argument
461 		container_of(port, struct sprd_uart_port, port);  in sprd_start_dma_rx()
469 	sprd_rx_full_thld(port, SPRD_RX_FIFO_FULL);  in sprd_start_dma_rx()
470 	ret = sprd_rx_dma_config(port, SPRD_RX_DMA_STEP);  in sprd_start_dma_rx()
474 	return sprd_uart_dma_submit(port, &sp->rx_dma, SPRD_UART_RX_SIZE,  in sprd_start_dma_rx()
478 static void sprd_release_dma(struct uart_port *port)  in sprd_release_dma()  argument
481 		container_of(port, struct sprd_uart_port, port);  in sprd_release_dma()
483 	sprd_uart_dma_enable(port, false);  in sprd_release_dma()
495 static void sprd_request_dma(struct uart_port *port)  in sprd_request_dma()  argument
498 		container_of(port, struct sprd_uart_port, port);  in sprd_request_dma()
503 	sp->tx_dma.chn = dma_request_chan(port->dev, "tx");  in sprd_request_dma()
505 		dev_err(port->dev, "request TX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
510 	sp->rx_dma.chn = dma_request_chan(port->dev, "rx");  in sprd_request_dma()
512 		dev_err(port->dev, "request RX DMA channel failed, ret = %ld\n",  in sprd_request_dma()
518 static void sprd_stop_tx(struct uart_port *port)  in sprd_stop_tx()  argument
520 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_stop_tx()
521 						 port);  in sprd_stop_tx()
525 		sprd_stop_tx_dma(port);  in sprd_stop_tx()
529 	iclr = serial_in(port, SPRD_ICLR);  in sprd_stop_tx()
530 	ien = serial_in(port, SPRD_IEN);  in sprd_stop_tx()
535 	serial_out(port, SPRD_IEN, ien);  in sprd_stop_tx()
536 	serial_out(port, SPRD_ICLR, iclr);  in sprd_stop_tx()
539 static void sprd_start_tx(struct uart_port *port)  in sprd_start_tx()  argument
541 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_start_tx()
542 						 port);  in sprd_start_tx()
546 		sprd_start_tx_dma(port);  in sprd_start_tx()
550 	ien = serial_in(port, SPRD_IEN);  in sprd_start_tx()
553 		serial_out(port, SPRD_IEN, ien);  in sprd_start_tx()
558 static void sprd_break_ctl(struct uart_port *port, int break_state)  in sprd_break_ctl()  argument
563 static int handle_lsr_errors(struct uart_port *port,  in handle_lsr_errors()  argument
572 		port->icount.brk++;  in handle_lsr_errors()
573 		ret = uart_handle_break(port);  in handle_lsr_errors()
577 		port->icount.parity++;  in handle_lsr_errors()
579 		port->icount.frame++;  in handle_lsr_errors()
581 		port->icount.overrun++;  in handle_lsr_errors()
584 	*lsr &= port->read_status_mask;  in handle_lsr_errors()
595 static inline void sprd_rx(struct uart_port *port)  in sprd_rx()  argument
597 	struct sprd_uart_port *sp = container_of(port, struct sprd_uart_port,  in sprd_rx()
598 						 port);  in sprd_rx()
599 	struct tty_port *tty = &port->state->port;  in sprd_rx()
603 		sprd_uart_dma_irq(port);  in sprd_rx()
607 	while ((serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK) &&  in sprd_rx()
609 		lsr = serial_in(port, SPRD_LSR);  in sprd_rx()
610 		ch = serial_in(port, SPRD_RXD);  in sprd_rx()
612 		port->icount.rx++;  in sprd_rx()
616 			if (handle_lsr_errors(port, &flag, &lsr))  in sprd_rx()
618 		if (uart_handle_sysrq_char(port, ch))  in sprd_rx()
621 		uart_insert_char(port, lsr, SPRD_LSR_OE, ch, flag);  in sprd_rx()
627 static inline void sprd_tx(struct uart_port *port)  in sprd_tx()  argument
629 	struct circ_buf *xmit = &port->state->xmit;  in sprd_tx()
632 	if (port->x_char) {  in sprd_tx()
633 		serial_out(port, SPRD_TXD, port->x_char);  in sprd_tx()
634 		port->icount.tx++;  in sprd_tx()
635 		port->x_char = 0;  in sprd_tx()
639 	if (uart_circ_empty(xmit) || uart_tx_stopped(port)) {  in sprd_tx()
640 		sprd_stop_tx(port);  in sprd_tx()
646 		serial_out(port, SPRD_TXD, xmit->buf[xmit->tail]);  in sprd_tx()
648 		port->icount.tx++;  in sprd_tx()
654 		uart_write_wakeup(port);  in sprd_tx()
657 		sprd_stop_tx(port);  in sprd_tx()
660 /* this handles the interrupt from one port */
663 	struct uart_port *port = dev_id;  in sprd_handle_irq()  local
666 	spin_lock(&port->lock);  in sprd_handle_irq()
668 	ims = serial_in(port, SPRD_IMSR);  in sprd_handle_irq()
671 		spin_unlock(&port->lock);  in sprd_handle_irq()
676 		serial_out(port, SPRD_ICLR, SPRD_ICLR_TIMEOUT);  in sprd_handle_irq()
679 		serial_out(port, SPRD_ICLR, SPRD_IMSR_BREAK_DETECT);  in sprd_handle_irq()
683 		sprd_rx(port);  in sprd_handle_irq()
686 		sprd_tx(port);  in sprd_handle_irq()
688 	spin_unlock(&port->lock);  in sprd_handle_irq()
693 static void sprd_uart_dma_startup(struct uart_port *port,  in sprd_uart_dma_startup()  argument
698 	sprd_request_dma(port);  in sprd_uart_dma_startup()
702 	ret = sprd_start_dma_rx(port);  in sprd_uart_dma_startup()
706 		dev_warn(port->dev, "fail to start RX dma mode\n");  in sprd_uart_dma_startup()
709 	sprd_uart_dma_enable(port, true);  in sprd_uart_dma_startup()
712 static int sprd_startup(struct uart_port *port)  in sprd_startup()  argument
720 	serial_out(port, SPRD_CTL2,  in sprd_startup()
725 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK)  in sprd_startup()
726 		serial_in(port, SPRD_RXD);  in sprd_startup()
730 	while (timeout-- && serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_startup()
734 	serial_out(port, SPRD_IEN, 0);  in sprd_startup()
735 	serial_out(port, SPRD_ICLR, ~0);  in sprd_startup()
738 	sp = container_of(port, struct sprd_uart_port, port);  in sprd_startup()
739 	snprintf(sp->name, sizeof(sp->name), "sprd_serial%d", port->line);  in sprd_startup()
741 	sprd_uart_dma_startup(port, sp);  in sprd_startup()
743 	ret = devm_request_irq(port->dev, port->irq, sprd_handle_irq,  in sprd_startup()
744 			       IRQF_SHARED, sp->name, port);  in sprd_startup()
746 		dev_err(port->dev, "fail to request serial irq %d, ret=%d\n",  in sprd_startup()
747 			port->irq, ret);  in sprd_startup()
750 	fc = serial_in(port, SPRD_CTL1);  in sprd_startup()
752 	serial_out(port, SPRD_CTL1, fc);  in sprd_startup()
755 	spin_lock_irqsave(&port->lock, flags);  in sprd_startup()
756 	ien = serial_in(port, SPRD_IEN);  in sprd_startup()
760 	serial_out(port, SPRD_IEN, ien);  in sprd_startup()
761 	spin_unlock_irqrestore(&port->lock, flags);  in sprd_startup()
766 static void sprd_shutdown(struct uart_port *port)  in sprd_shutdown()  argument
768 	sprd_release_dma(port);  in sprd_shutdown()
769 	serial_out(port, SPRD_IEN, 0);  in sprd_shutdown()
770 	serial_out(port, SPRD_ICLR, ~0);  in sprd_shutdown()
771 	devm_free_irq(port->dev, port->irq, port);  in sprd_shutdown()
774 static void sprd_set_termios(struct uart_port *port, struct ktermios *termios,  in sprd_set_termios()  argument
782 	baud = uart_get_baud_rate(port, termios, old, 0, SPRD_BAUD_IO_LIMIT);  in sprd_set_termios()
784 	quot = port->uartclk / baud;  in sprd_set_termios()
821 	spin_lock_irqsave(&port->lock, flags);  in sprd_set_termios()
823 	/* update the per-port timeout */  in sprd_set_termios()
824 	uart_update_timeout(port, termios->c_cflag, baud);  in sprd_set_termios()
826 	port->read_status_mask = SPRD_LSR_OE;  in sprd_set_termios()
828 		port->read_status_mask |= SPRD_LSR_FE | SPRD_LSR_PE;  in sprd_set_termios()
830 		port->read_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
833 	port->ignore_status_mask = 0;  in sprd_set_termios()
835 		port->ignore_status_mask |= SPRD_LSR_PE | SPRD_LSR_FE;  in sprd_set_termios()
837 		port->ignore_status_mask |= SPRD_LSR_BI;  in sprd_set_termios()
843 			port->ignore_status_mask |= SPRD_LSR_OE;  in sprd_set_termios()
847 	fc = serial_in(port, SPRD_CTL1);  in sprd_set_termios()
856 	serial_out(port, SPRD_CLKD0, quot & SPRD_CLKD0_MASK);  in sprd_set_termios()
859 	serial_out(port, SPRD_CLKD1,  in sprd_set_termios()
861 	serial_out(port, SPRD_LCR, lcr);  in sprd_set_termios()
863 	serial_out(port, SPRD_CTL1, fc);  in sprd_set_termios()
865 	spin_unlock_irqrestore(&port->lock, flags);  in sprd_set_termios()
872 static const char *sprd_type(struct uart_port *port)  in sprd_type()  argument
877 static void sprd_release_port(struct uart_port *port)  in sprd_release_port()  argument
882 static int sprd_request_port(struct uart_port *port)  in sprd_request_port()  argument
887 static void sprd_config_port(struct uart_port *port, int flags)  in sprd_config_port()  argument
890 		port->type = PORT_SPRD;  in sprd_config_port()
893 static int sprd_verify_port(struct uart_port *port, struct serial_struct *ser)  in sprd_verify_port()  argument
897 	if (port->irq != ser->irq)  in sprd_verify_port()
899 	if (port->iotype != ser->io_type)  in sprd_verify_port()
904 static void sprd_pm(struct uart_port *port, unsigned int state,  in sprd_pm()  argument
908 		container_of(port, struct sprd_uart_port, port);  in sprd_pm()
921 static int sprd_poll_init(struct uart_port *port)  in sprd_poll_init()  argument
923 	if (port->state->pm_state != UART_PM_STATE_ON) {  in sprd_poll_init()
924 		sprd_pm(port, UART_PM_STATE_ON, 0);  in sprd_poll_init()
925 		port->state->pm_state = UART_PM_STATE_ON;  in sprd_poll_init()
931 static int sprd_poll_get_char(struct uart_port *port)  in sprd_poll_get_char()  argument
933 	while (!(serial_in(port, SPRD_STS1) & SPRD_RX_FIFO_CNT_MASK))  in sprd_poll_get_char()
936 	return serial_in(port, SPRD_RXD);  in sprd_poll_get_char()
939 static void sprd_poll_put_char(struct uart_port *port, unsigned char ch)  in sprd_poll_put_char()  argument
941 	while (serial_in(port, SPRD_STS1) & SPRD_TX_FIFO_CNT_MASK)  in sprd_poll_put_char()
944 	serial_out(port, SPRD_TXD, ch);  in sprd_poll_put_char()
973 static void wait_for_xmitr(struct uart_port *port)  in wait_for_xmitr()  argument
979 		status = serial_in(port, SPRD_STS1);  in wait_for_xmitr()
986 static void sprd_console_putchar(struct uart_port *port, unsigned char ch)  in sprd_console_putchar()  argument
988 	wait_for_xmitr(port);  in sprd_console_putchar()
989 	serial_out(port, SPRD_TXD, ch);  in sprd_console_putchar()
995 	struct uart_port *port = &sprd_port[co->index]->port;  in sprd_console_write()  local
999 	if (port->sysrq)  in sprd_console_write()
1002 		locked = spin_trylock_irqsave(&port->lock, flags);  in sprd_console_write()
1004 		spin_lock_irqsave(&port->lock, flags);  in sprd_console_write()
1006 	uart_console_write(port, s, count, sprd_console_putchar);  in sprd_console_write()
1009 	wait_for_xmitr(port);  in sprd_console_write()
1012 		spin_unlock_irqrestore(&port->lock, flags);  in sprd_console_write()
1027 	if (!sprd_uart_port || !sprd_uart_port->port.membase) {  in sprd_console_setup()
1028 		pr_info("serial port %d not yet initialized\n", co->index);  in sprd_console_setup()
1035 	return uart_set_options(&sprd_uart_port->port, co, baud,  in sprd_console_setup()
1060 static void sprd_putc(struct uart_port *port, unsigned char c)  in sprd_putc()  argument
1065 	       !(readl(port->membase + SPRD_LSR) & SPRD_LSR_TX_OVER))  in sprd_putc()
1068 	writeb(c, port->membase + SPRD_TXD);  in sprd_putc()
1075 	uart_console_write(&dev->port, s, n, sprd_putc);  in sprd_early_write()
1081 	if (!device->port.membase)  in sprd_early_console_setup()
1109 		uart_remove_one_port(&sprd_uart_driver, &sup->port);  in sprd_remove()
1110 		sprd_port[sup->port.line] = NULL;  in sprd_remove()
1193 	up = &sprd_port[index]->port;  in sprd_probe()
1251 	uart_suspend_port(&sprd_uart_driver, &sup->port);  in sprd_suspend()
1260 	uart_resume_port(&sprd_uart_driver, &sup->port);  in sprd_resume()