152 static void tegra_uart_start_next_tx(struct tegra_uart_port *tup);
153 static int tegra_uart_start_rx_dma(struct tegra_uart_port *tup);
154 static void tegra_uart_dma_channel_free(struct tegra_uart_port *tup,
157 static inline unsigned long tegra_uart_read(struct tegra_uart_port *tup,  in tegra_uart_read()  argument
160 	return readl(tup->uport.membase + (reg << tup->uport.regshift));  in tegra_uart_read()
163 static inline void tegra_uart_write(struct tegra_uart_port *tup, unsigned val,  in tegra_uart_write()  argument
166 	writel(val, tup->uport.membase + (reg << tup->uport.regshift));  in tegra_uart_write()
176 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_get_mctrl()  local
187 	if (tup->enable_modem_interrupt)  in tegra_uart_get_mctrl()
192 static void set_rts(struct tegra_uart_port *tup, bool active)  in set_rts()  argument
196 	mcr = tup->mcr_shadow;  in set_rts()
201 	if (mcr != tup->mcr_shadow) {  in set_rts()
202 		tegra_uart_write(tup, mcr, UART_MCR);  in set_rts()
203 		tup->mcr_shadow = mcr;  in set_rts()
207 static void set_dtr(struct tegra_uart_port *tup, bool active)  in set_dtr()  argument
211 	mcr = tup->mcr_shadow;  in set_dtr()
216 	if (mcr != tup->mcr_shadow) {  in set_dtr()
217 		tegra_uart_write(tup, mcr, UART_MCR);  in set_dtr()
218 		tup->mcr_shadow = mcr;  in set_dtr()
222 static void set_loopbk(struct tegra_uart_port *tup, bool active)  in set_loopbk()  argument
224 	unsigned long mcr = tup->mcr_shadow;  in set_loopbk()
231 	if (mcr != tup->mcr_shadow) {  in set_loopbk()
232 		tegra_uart_write(tup, mcr, UART_MCR);  in set_loopbk()
233 		tup->mcr_shadow = mcr;  in set_loopbk()
239 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_set_mctrl()  local
242 	tup->rts_active = !!(mctrl & TIOCM_RTS);  in tegra_uart_set_mctrl()
243 	set_rts(tup, tup->rts_active);  in tegra_uart_set_mctrl()
246 	set_dtr(tup, enable);  in tegra_uart_set_mctrl()
249 	set_loopbk(tup, enable);  in tegra_uart_set_mctrl()
254 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_break_ctl()  local
257 	lcr = tup->lcr_shadow;  in tegra_uart_break_ctl()
262 	tegra_uart_write(tup, lcr, UART_LCR);  in tegra_uart_break_ctl()
263 	tup->lcr_shadow = lcr;  in tegra_uart_break_ctl()
275 static void tegra_uart_wait_cycle_time(struct tegra_uart_port *tup,  in tegra_uart_wait_cycle_time()  argument
278 	if (tup->current_baud)  in tegra_uart_wait_cycle_time()
279 		udelay(DIV_ROUND_UP(cycles * 1000000, tup->current_baud * 16));  in tegra_uart_wait_cycle_time()
283 static void tegra_uart_wait_sym_time(struct tegra_uart_port *tup,  in tegra_uart_wait_sym_time()  argument
286 	if (tup->current_baud)  in tegra_uart_wait_sym_time()
287 		udelay(DIV_ROUND_UP(syms * tup->symb_bit * 1000000,  in tegra_uart_wait_sym_time()
288 			tup->current_baud));  in tegra_uart_wait_sym_time()
291 static int tegra_uart_wait_fifo_mode_enabled(struct tegra_uart_port *tup)  in tegra_uart_wait_fifo_mode_enabled()  argument
297 		iir = tegra_uart_read(tup, UART_IIR);  in tegra_uart_wait_fifo_mode_enabled()
306 static void tegra_uart_fifo_reset(struct tegra_uart_port *tup, u8 fcr_bits)  in tegra_uart_fifo_reset()  argument
308 	unsigned long fcr = tup->fcr_shadow;  in tegra_uart_fifo_reset()
311 	if (tup->rts_active)  in tegra_uart_fifo_reset()
312 		set_rts(tup, false);  in tegra_uart_fifo_reset()
314 	if (tup->cdata->allow_txfifo_reset_fifo_mode) {  in tegra_uart_fifo_reset()
316 		tegra_uart_write(tup, fcr, UART_FCR);  in tegra_uart_fifo_reset()
319 		tegra_uart_write(tup, fcr, UART_FCR);  in tegra_uart_fifo_reset()
322 		tegra_uart_write(tup, fcr, UART_FCR);  in tegra_uart_fifo_reset()
324 		tegra_uart_write(tup, fcr, UART_FCR);  in tegra_uart_fifo_reset()
325 		if (tup->cdata->fifo_mode_enable_status)  in tegra_uart_fifo_reset()
326 			tegra_uart_wait_fifo_mode_enabled(tup);  in tegra_uart_fifo_reset()
330 	tegra_uart_read(tup, UART_SCR);  in tegra_uart_fifo_reset()
337 	tegra_uart_wait_cycle_time(tup, 32);  in tegra_uart_fifo_reset()
340 		lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_fifo_reset()
346 	if (tup->rts_active)  in tegra_uart_fifo_reset()
347 		set_rts(tup, true);  in tegra_uart_fifo_reset()
350 static long tegra_get_tolerance_rate(struct tegra_uart_port *tup,  in tegra_get_tolerance_rate()  argument
355 	for (i = 0; i < tup->n_adjustable_baud_rates; ++i) {  in tegra_get_tolerance_rate()
356 		if (baud >= tup->baud_tolerance[i].lower_range_baud &&  in tegra_get_tolerance_rate()
357 		    baud <= tup->baud_tolerance[i].upper_range_baud)  in tegra_get_tolerance_rate()
359 				tup->baud_tolerance[i].tolerance) / 10000);  in tegra_get_tolerance_rate()
365 static int tegra_check_rate_in_range(struct tegra_uart_port *tup)  in tegra_check_rate_in_range()  argument
369 	diff = ((long)(tup->configured_rate - tup->required_rate) * 10000)  in tegra_check_rate_in_range()
370 		/ tup->required_rate;  in tegra_check_rate_in_range()
371 	if (diff < (tup->cdata->error_tolerance_low_range * 100) ||  in tegra_check_rate_in_range()
372 	    diff > (tup->cdata->error_tolerance_high_range * 100)) {  in tegra_check_rate_in_range()
373 		dev_err(tup->uport.dev,  in tegra_check_rate_in_range()
381 static int tegra_set_baudrate(struct tegra_uart_port *tup, unsigned int baud)  in tegra_set_baudrate()  argument
389 	if (tup->current_baud == baud)  in tegra_set_baudrate()
392 	if (tup->cdata->support_clk_src_div) {  in tegra_set_baudrate()
394 		tup->required_rate = rate;  in tegra_set_baudrate()
396 		if (tup->n_adjustable_baud_rates)  in tegra_set_baudrate()
397 			rate = tegra_get_tolerance_rate(tup, baud, rate);  in tegra_set_baudrate()
399 		ret = clk_set_rate(tup->uart_clk, rate);  in tegra_set_baudrate()
401 			dev_err(tup->uport.dev,  in tegra_set_baudrate()
405 		tup->configured_rate = clk_get_rate(tup->uart_clk);  in tegra_set_baudrate()
407 		ret = tegra_check_rate_in_range(tup);  in tegra_set_baudrate()
411 		rate = clk_get_rate(tup->uart_clk);  in tegra_set_baudrate()
415 	spin_lock_irqsave(&tup->uport.lock, flags);  in tegra_set_baudrate()
416 	lcr = tup->lcr_shadow;  in tegra_set_baudrate()
418 	tegra_uart_write(tup, lcr, UART_LCR);  in tegra_set_baudrate()
420 	tegra_uart_write(tup, divisor & 0xFF, UART_TX);  in tegra_set_baudrate()
421 	tegra_uart_write(tup, ((divisor >> 8) & 0xFF), UART_IER);  in tegra_set_baudrate()
424 	tegra_uart_write(tup, lcr, UART_LCR);  in tegra_set_baudrate()
427 	tegra_uart_read(tup, UART_SCR);  in tegra_set_baudrate()
428 	spin_unlock_irqrestore(&tup->uport.lock, flags);  in tegra_set_baudrate()
430 	tup->current_baud = baud;  in tegra_set_baudrate()
433 	tegra_uart_wait_sym_time(tup, 2);  in tegra_set_baudrate()
437 static char tegra_uart_decode_rx_error(struct tegra_uart_port *tup,  in tegra_uart_decode_rx_error()  argument
446 			tup->uport.icount.overrun++;  in tegra_uart_decode_rx_error()
447 			dev_dbg(tup->uport.dev, "Got overrun errors\n");  in tegra_uart_decode_rx_error()
451 			tup->uport.icount.parity++;  in tegra_uart_decode_rx_error()
452 			dev_dbg(tup->uport.dev, "Got Parity errors\n");  in tegra_uart_decode_rx_error()
455 			tup->uport.icount.frame++;  in tegra_uart_decode_rx_error()
456 			dev_dbg(tup->uport.dev, "Got frame errors\n");  in tegra_uart_decode_rx_error()
463 				tegra_uart_fifo_reset(tup, UART_FCR_CLEAR_RCVR);  in tegra_uart_decode_rx_error()
464 			if (tup->uport.ignore_status_mask & UART_LSR_BI)  in tegra_uart_decode_rx_error()
467 			tup->uport.icount.brk++;  in tegra_uart_decode_rx_error()
468 			dev_dbg(tup->uport.dev, "Got Break\n");  in tegra_uart_decode_rx_error()
470 		uart_insert_char(&tup->uport, lsr, UART_LSR_OE, 0, flag);  in tegra_uart_decode_rx_error()
486 static void tegra_uart_fill_tx_fifo(struct tegra_uart_port *tup, int max_bytes)  in tegra_uart_fill_tx_fifo()  argument
488 	struct circ_buf *xmit = &tup->uport.state->xmit;  in tegra_uart_fill_tx_fifo()
493 		if (tup->cdata->tx_fifo_full_status) {  in tegra_uart_fill_tx_fifo()
494 			unsigned long lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_fill_tx_fifo()
498 		tegra_uart_write(tup, xmit->buf[xmit->tail], UART_TX);  in tegra_uart_fill_tx_fifo()
500 		tup->uport.icount.tx++;  in tegra_uart_fill_tx_fifo()
504 static void tegra_uart_start_pio_tx(struct tegra_uart_port *tup,  in tegra_uart_start_pio_tx()  argument
510 	tup->tx_in_progress = TEGRA_UART_TX_PIO;  in tegra_uart_start_pio_tx()
511 	tup->tx_bytes = bytes;  in tegra_uart_start_pio_tx()
512 	tup->ier_shadow |= UART_IER_THRI;  in tegra_uart_start_pio_tx()
513 	tegra_uart_write(tup, tup->ier_shadow, UART_IER);  in tegra_uart_start_pio_tx()
518 	struct tegra_uart_port *tup = args;  in tegra_uart_tx_dma_complete()  local
519 	struct circ_buf *xmit = &tup->uport.state->xmit;  in tegra_uart_tx_dma_complete()
524 	dmaengine_tx_status(tup->tx_dma_chan, tup->tx_cookie, &state);  in tegra_uart_tx_dma_complete()
525 	count = tup->tx_bytes_requested - state.residue;  in tegra_uart_tx_dma_complete()
526 	async_tx_ack(tup->tx_dma_desc);  in tegra_uart_tx_dma_complete()
527 	spin_lock_irqsave(&tup->uport.lock, flags);  in tegra_uart_tx_dma_complete()
528 	uart_xmit_advance(&tup->uport, count);  in tegra_uart_tx_dma_complete()
529 	tup->tx_in_progress = 0;  in tegra_uart_tx_dma_complete()
531 		uart_write_wakeup(&tup->uport);  in tegra_uart_tx_dma_complete()
532 	tegra_uart_start_next_tx(tup);  in tegra_uart_tx_dma_complete()
533 	spin_unlock_irqrestore(&tup->uport.lock, flags);  in tegra_uart_tx_dma_complete()
536 static int tegra_uart_start_tx_dma(struct tegra_uart_port *tup,  in tegra_uart_start_tx_dma()  argument
539 	struct circ_buf *xmit = &tup->uport.state->xmit;  in tegra_uart_start_tx_dma()
542 	tup->tx_bytes = count & ~(0xF);  in tegra_uart_start_tx_dma()
543 	tx_phys_addr = tup->tx_dma_buf_phys + xmit->tail;  in tegra_uart_start_tx_dma()
545 	dma_sync_single_for_device(tup->uport.dev, tx_phys_addr,  in tegra_uart_start_tx_dma()
546 				   tup->tx_bytes, DMA_TO_DEVICE);  in tegra_uart_start_tx_dma()
548 	tup->tx_dma_desc = dmaengine_prep_slave_single(tup->tx_dma_chan,  in tegra_uart_start_tx_dma()
549 				tx_phys_addr, tup->tx_bytes, DMA_MEM_TO_DEV,  in tegra_uart_start_tx_dma()
551 	if (!tup->tx_dma_desc) {  in tegra_uart_start_tx_dma()
552 		dev_err(tup->uport.dev, "Not able to get desc for Tx\n");  in tegra_uart_start_tx_dma()
556 	tup->tx_dma_desc->callback = tegra_uart_tx_dma_complete;  in tegra_uart_start_tx_dma()
557 	tup->tx_dma_desc->callback_param = tup;  in tegra_uart_start_tx_dma()
558 	tup->tx_in_progress = TEGRA_UART_TX_DMA;  in tegra_uart_start_tx_dma()
559 	tup->tx_bytes_requested = tup->tx_bytes;  in tegra_uart_start_tx_dma()
560 	tup->tx_cookie = dmaengine_submit(tup->tx_dma_desc);  in tegra_uart_start_tx_dma()
561 	dma_async_issue_pending(tup->tx_dma_chan);  in tegra_uart_start_tx_dma()
565 static void tegra_uart_start_next_tx(struct tegra_uart_port *tup)  in tegra_uart_start_next_tx()  argument
569 	struct circ_buf *xmit = &tup->uport.state->xmit;  in tegra_uart_start_next_tx()
571 	if (!tup->current_baud)  in tegra_uart_start_next_tx()
579 	if (tup->use_tx_pio || count < TEGRA_UART_MIN_DMA)  in tegra_uart_start_next_tx()
580 		tegra_uart_start_pio_tx(tup, count);  in tegra_uart_start_next_tx()
582 		tegra_uart_start_pio_tx(tup, BYTES_TO_ALIGN(tail));  in tegra_uart_start_next_tx()
584 		tegra_uart_start_tx_dma(tup, count);  in tegra_uart_start_next_tx()
590 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_start_tx()  local
593 	if (!uart_circ_empty(xmit) && !tup->tx_in_progress)  in tegra_uart_start_tx()
594 		tegra_uart_start_next_tx(tup);  in tegra_uart_start_tx()
599 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_tx_empty()  local
604 	if (!tup->tx_in_progress) {  in tegra_uart_tx_empty()
605 		unsigned long lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_tx_empty()
615 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_stop_tx()  local
619 	if (tup->tx_in_progress != TEGRA_UART_TX_DMA)  in tegra_uart_stop_tx()
622 	dmaengine_terminate_all(tup->tx_dma_chan);  in tegra_uart_stop_tx()
623 	dmaengine_tx_status(tup->tx_dma_chan, tup->tx_cookie, &state);  in tegra_uart_stop_tx()
624 	count = tup->tx_bytes_requested - state.residue;  in tegra_uart_stop_tx()
625 	async_tx_ack(tup->tx_dma_desc);  in tegra_uart_stop_tx()
626 	uart_xmit_advance(&tup->uport, count);  in tegra_uart_stop_tx()
627 	tup->tx_in_progress = 0;  in tegra_uart_stop_tx()
630 static void tegra_uart_handle_tx_pio(struct tegra_uart_port *tup)  in tegra_uart_handle_tx_pio()  argument
632 	struct circ_buf *xmit = &tup->uport.state->xmit;  in tegra_uart_handle_tx_pio()
634 	tegra_uart_fill_tx_fifo(tup, tup->tx_bytes);  in tegra_uart_handle_tx_pio()
635 	tup->tx_in_progress = 0;  in tegra_uart_handle_tx_pio()
637 		uart_write_wakeup(&tup->uport);  in tegra_uart_handle_tx_pio()
638 	tegra_uart_start_next_tx(tup);  in tegra_uart_handle_tx_pio()
641 static void tegra_uart_handle_rx_pio(struct tegra_uart_port *tup,  in tegra_uart_handle_rx_pio()  argument
649 		lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_handle_rx_pio()
653 		flag = tegra_uart_decode_rx_error(tup, lsr);  in tegra_uart_handle_rx_pio()
657 		ch = (unsigned char) tegra_uart_read(tup, UART_RX);  in tegra_uart_handle_rx_pio()
658 		tup->uport.icount.rx++;  in tegra_uart_handle_rx_pio()
660 		if (uart_handle_sysrq_char(&tup->uport, ch))  in tegra_uart_handle_rx_pio()
663 		if (tup->uport.ignore_status_mask & UART_LSR_DR)  in tegra_uart_handle_rx_pio()
670 static void tegra_uart_copy_rx_to_tty(struct tegra_uart_port *tup,  in tegra_uart_copy_rx_to_tty()  argument
680 	tup->uport.icount.rx += count;  in tegra_uart_copy_rx_to_tty()
682 	if (tup->uport.ignore_status_mask & UART_LSR_DR)  in tegra_uart_copy_rx_to_tty()
685 	dma_sync_single_for_cpu(tup->uport.dev, tup->rx_dma_buf_phys,  in tegra_uart_copy_rx_to_tty()
688 			((unsigned char *)(tup->rx_dma_buf_virt)), count);  in tegra_uart_copy_rx_to_tty()
691 		dev_err(tup->uport.dev, "RxData copy to tty layer failed\n");  in tegra_uart_copy_rx_to_tty()
693 	dma_sync_single_for_device(tup->uport.dev, tup->rx_dma_buf_phys,  in tegra_uart_copy_rx_to_tty()
697 static void do_handle_rx_pio(struct tegra_uart_port *tup)  in do_handle_rx_pio()  argument
699 	struct tty_struct *tty = tty_port_tty_get(&tup->uport.state->port);  in do_handle_rx_pio()
700 	struct tty_port *port = &tup->uport.state->port;  in do_handle_rx_pio()
702 	tegra_uart_handle_rx_pio(tup, port);  in do_handle_rx_pio()
709 static void tegra_uart_rx_buffer_push(struct tegra_uart_port *tup,  in tegra_uart_rx_buffer_push()  argument
712 	struct tty_port *port = &tup->uport.state->port;  in tegra_uart_rx_buffer_push()
715 	async_tx_ack(tup->rx_dma_desc);  in tegra_uart_rx_buffer_push()
716 	count = tup->rx_bytes_requested - residue;  in tegra_uart_rx_buffer_push()
719 	tegra_uart_copy_rx_to_tty(tup, port, count);  in tegra_uart_rx_buffer_push()
721 	do_handle_rx_pio(tup);  in tegra_uart_rx_buffer_push()
726 	struct tegra_uart_port *tup = args;  in tegra_uart_rx_dma_complete()  local
727 	struct uart_port *u = &tup->uport;  in tegra_uart_rx_dma_complete()
734 	status = dmaengine_tx_status(tup->rx_dma_chan, tup->rx_cookie, &state);  in tegra_uart_rx_dma_complete()
737 		dev_dbg(tup->uport.dev, "RX DMA is in progress\n");  in tegra_uart_rx_dma_complete()
742 	if (tup->rts_active)  in tegra_uart_rx_dma_complete()
743 		set_rts(tup, false);  in tegra_uart_rx_dma_complete()
745 	tup->rx_dma_active = false;  in tegra_uart_rx_dma_complete()
746 	tegra_uart_rx_buffer_push(tup, 0);  in tegra_uart_rx_dma_complete()
747 	tegra_uart_start_rx_dma(tup);  in tegra_uart_rx_dma_complete()
750 	if (tup->rts_active)  in tegra_uart_rx_dma_complete()
751 		set_rts(tup, true);  in tegra_uart_rx_dma_complete()
757 static void tegra_uart_terminate_rx_dma(struct tegra_uart_port *tup)  in tegra_uart_terminate_rx_dma()  argument
761 	if (!tup->rx_dma_active) {  in tegra_uart_terminate_rx_dma()
762 		do_handle_rx_pio(tup);  in tegra_uart_terminate_rx_dma()
766 	dmaengine_terminate_all(tup->rx_dma_chan);  in tegra_uart_terminate_rx_dma()
767 	dmaengine_tx_status(tup->rx_dma_chan, tup->rx_cookie, &state);  in tegra_uart_terminate_rx_dma()
769 	tegra_uart_rx_buffer_push(tup, state.residue);  in tegra_uart_terminate_rx_dma()
770 	tup->rx_dma_active = false;  in tegra_uart_terminate_rx_dma()
773 static void tegra_uart_handle_rx_dma(struct tegra_uart_port *tup)  in tegra_uart_handle_rx_dma()  argument
776 	if (tup->rts_active)  in tegra_uart_handle_rx_dma()
777 		set_rts(tup, false);  in tegra_uart_handle_rx_dma()
779 	tegra_uart_terminate_rx_dma(tup);  in tegra_uart_handle_rx_dma()
781 	if (tup->rts_active)  in tegra_uart_handle_rx_dma()
782 		set_rts(tup, true);  in tegra_uart_handle_rx_dma()
785 static int tegra_uart_start_rx_dma(struct tegra_uart_port *tup)  in tegra_uart_start_rx_dma()  argument
789 	if (tup->rx_dma_active)  in tegra_uart_start_rx_dma()
792 	tup->rx_dma_desc = dmaengine_prep_slave_single(tup->rx_dma_chan,  in tegra_uart_start_rx_dma()
793 				tup->rx_dma_buf_phys, count, DMA_DEV_TO_MEM,  in tegra_uart_start_rx_dma()
795 	if (!tup->rx_dma_desc) {  in tegra_uart_start_rx_dma()
796 		dev_err(tup->uport.dev, "Not able to get desc for Rx\n");  in tegra_uart_start_rx_dma()
800 	tup->rx_dma_active = true;  in tegra_uart_start_rx_dma()
801 	tup->rx_dma_desc->callback = tegra_uart_rx_dma_complete;  in tegra_uart_start_rx_dma()
802 	tup->rx_dma_desc->callback_param = tup;  in tegra_uart_start_rx_dma()
803 	tup->rx_bytes_requested = count;  in tegra_uart_start_rx_dma()
804 	tup->rx_cookie = dmaengine_submit(tup->rx_dma_desc);  in tegra_uart_start_rx_dma()
805 	dma_async_issue_pending(tup->rx_dma_chan);  in tegra_uart_start_rx_dma()
811 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_handle_modem_signal_change()  local
814 	msr = tegra_uart_read(tup, UART_MSR);  in tegra_uart_handle_modem_signal_change()
819 		tup->uport.icount.rng++;  in tegra_uart_handle_modem_signal_change()
821 		tup->uport.icount.dsr++;  in tegra_uart_handle_modem_signal_change()
824 		uart_handle_dcd_change(&tup->uport, msr & UART_MSR_DCD);  in tegra_uart_handle_modem_signal_change()
827 		uart_handle_cts_change(&tup->uport, msr & UART_MSR_CTS);  in tegra_uart_handle_modem_signal_change()
832 	struct tegra_uart_port *tup = data;  in tegra_uart_isr()  local
833 	struct uart_port *u = &tup->uport;  in tegra_uart_isr()
842 		iir = tegra_uart_read(tup, UART_IIR);  in tegra_uart_isr()
844 			if (!tup->use_rx_pio && is_rx_int) {  in tegra_uart_isr()
845 				tegra_uart_handle_rx_dma(tup);  in tegra_uart_isr()
846 				if (tup->rx_in_progress) {  in tegra_uart_isr()
847 					ier = tup->ier_shadow;  in tegra_uart_isr()
850 					tup->ier_shadow = ier;  in tegra_uart_isr()
851 					tegra_uart_write(tup, ier, UART_IER);  in tegra_uart_isr()
854 				tegra_uart_start_rx_dma(tup);  in tegra_uart_isr()
866 			tup->ier_shadow &= ~UART_IER_THRI;  in tegra_uart_isr()
867 			tegra_uart_write(tup, tup->ier_shadow, UART_IER);  in tegra_uart_isr()
868 			tegra_uart_handle_tx_pio(tup);  in tegra_uart_isr()
873 			if (!tup->use_rx_pio) {  in tegra_uart_isr()
874 				is_rx_int = tup->rx_in_progress;  in tegra_uart_isr()
876 				ier = tup->ier_shadow;  in tegra_uart_isr()
879 				tup->ier_shadow = ier;  in tegra_uart_isr()
880 				tegra_uart_write(tup, ier, UART_IER);  in tegra_uart_isr()
885 			if (!tup->use_rx_pio) {  in tegra_uart_isr()
886 				is_rx_start = tup->rx_in_progress;  in tegra_uart_isr()
887 				tup->ier_shadow  &= ~UART_IER_RDI;  in tegra_uart_isr()
888 				tegra_uart_write(tup, tup->ier_shadow,  in tegra_uart_isr()
891 				do_handle_rx_pio(tup);  in tegra_uart_isr()
896 			tegra_uart_decode_rx_error(tup,  in tegra_uart_isr()
897 					tegra_uart_read(tup, UART_LSR));  in tegra_uart_isr()
909 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_stop_rx()  local
910 	struct tty_port *port = &tup->uport.state->port;  in tegra_uart_stop_rx()
913 	if (tup->rts_active)  in tegra_uart_stop_rx()
914 		set_rts(tup, false);  in tegra_uart_stop_rx()
916 	if (!tup->rx_in_progress)  in tegra_uart_stop_rx()
919 	tegra_uart_wait_sym_time(tup, 1); /* wait one character interval */  in tegra_uart_stop_rx()
921 	ier = tup->ier_shadow;  in tegra_uart_stop_rx()
924 	tup->ier_shadow = ier;  in tegra_uart_stop_rx()
925 	tegra_uart_write(tup, ier, UART_IER);  in tegra_uart_stop_rx()
926 	tup->rx_in_progress = 0;  in tegra_uart_stop_rx()
928 	if (!tup->use_rx_pio)  in tegra_uart_stop_rx()
929 		tegra_uart_terminate_rx_dma(tup);  in tegra_uart_stop_rx()
931 		tegra_uart_handle_rx_pio(tup, port);  in tegra_uart_stop_rx()
934 static void tegra_uart_hw_deinit(struct tegra_uart_port *tup)  in tegra_uart_hw_deinit()  argument
937 	unsigned long char_time = DIV_ROUND_UP(10000000, tup->current_baud);  in tegra_uart_hw_deinit()
938 	unsigned long fifo_empty_time = tup->uport.fifosize * char_time;  in tegra_uart_hw_deinit()
945 	tegra_uart_write(tup, 0, UART_IER);  in tegra_uart_hw_deinit()
947 	lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_hw_deinit()
949 		msr = tegra_uart_read(tup, UART_MSR);  in tegra_uart_hw_deinit()
950 		mcr = tegra_uart_read(tup, UART_MCR);  in tegra_uart_hw_deinit()
952 			dev_err(tup->uport.dev,  in tegra_uart_hw_deinit()
961 				msr = tegra_uart_read(tup, UART_MSR);  in tegra_uart_hw_deinit()
962 				mcr = tegra_uart_read(tup, UART_MCR);  in tegra_uart_hw_deinit()
965 					dev_err(tup->uport.dev,  in tegra_uart_hw_deinit()
969 			lsr = tegra_uart_read(tup, UART_LSR);  in tegra_uart_hw_deinit()
973 	spin_lock_irqsave(&tup->uport.lock, flags);  in tegra_uart_hw_deinit()
975 	tegra_uart_fifo_reset(tup, UART_FCR_CLEAR_XMIT | UART_FCR_CLEAR_RCVR);  in tegra_uart_hw_deinit()
976 	tup->current_baud = 0;  in tegra_uart_hw_deinit()
977 	spin_unlock_irqrestore(&tup->uport.lock, flags);  in tegra_uart_hw_deinit()
979 	tup->rx_in_progress = 0;  in tegra_uart_hw_deinit()
980 	tup->tx_in_progress = 0;  in tegra_uart_hw_deinit()
982 	if (!tup->use_rx_pio)  in tegra_uart_hw_deinit()
983 		tegra_uart_dma_channel_free(tup, true);  in tegra_uart_hw_deinit()
984 	if (!tup->use_tx_pio)  in tegra_uart_hw_deinit()
985 		tegra_uart_dma_channel_free(tup, false);  in tegra_uart_hw_deinit()
987 	clk_disable_unprepare(tup->uart_clk);  in tegra_uart_hw_deinit()
990 static int tegra_uart_hw_init(struct tegra_uart_port *tup)  in tegra_uart_hw_init()  argument
994 	tup->fcr_shadow = 0;  in tegra_uart_hw_init()
995 	tup->mcr_shadow = 0;  in tegra_uart_hw_init()
996 	tup->lcr_shadow = 0;  in tegra_uart_hw_init()
997 	tup->ier_shadow = 0;  in tegra_uart_hw_init()
998 	tup->current_baud = 0;  in tegra_uart_hw_init()
1000 	clk_prepare_enable(tup->uart_clk);  in tegra_uart_hw_init()
1003 	reset_control_assert(tup->rst);  in tegra_uart_hw_init()
1005 	reset_control_deassert(tup->rst);  in tegra_uart_hw_init()
1007 	tup->rx_in_progress = 0;  in tegra_uart_hw_init()
1008 	tup->tx_in_progress = 0;  in tegra_uart_hw_init()
1028 	tup->fcr_shadow = UART_FCR_ENABLE_FIFO;  in tegra_uart_hw_init()
1030 	if (tup->use_rx_pio) {  in tegra_uart_hw_init()
1031 		tup->fcr_shadow |= UART_FCR_R_TRIG_11;  in tegra_uart_hw_init()
1033 		if (tup->cdata->max_dma_burst_bytes == 8)  in tegra_uart_hw_init()
1034 			tup->fcr_shadow |= UART_FCR_R_TRIG_10;  in tegra_uart_hw_init()
1036 			tup->fcr_shadow |= UART_FCR_R_TRIG_01;  in tegra_uart_hw_init()
1039 	tup->fcr_shadow |= TEGRA_UART_TX_TRIG_16B;  in tegra_uart_hw_init()
1040 	tegra_uart_write(tup, tup->fcr_shadow, UART_FCR);  in tegra_uart_hw_init()
1043 	tegra_uart_read(tup, UART_SCR);  in tegra_uart_hw_init()
1045 	if (tup->cdata->fifo_mode_enable_status) {  in tegra_uart_hw_init()
1046 		ret = tegra_uart_wait_fifo_mode_enabled(tup);  in tegra_uart_hw_init()
1048 			dev_err(tup->uport.dev,  in tegra_uart_hw_init()
1059 		tegra_uart_wait_cycle_time(tup, 3);  in tegra_uart_hw_init()
1067 	ret = tegra_set_baudrate(tup, TEGRA_UART_DEFAULT_BAUD);  in tegra_uart_hw_init()
1069 		dev_err(tup->uport.dev, "Failed to set baud rate\n");  in tegra_uart_hw_init()
1072 	if (!tup->use_rx_pio) {  in tegra_uart_hw_init()
1073 		tup->lcr_shadow = TEGRA_UART_DEFAULT_LSR;  in tegra_uart_hw_init()
1074 		tup->fcr_shadow |= UART_FCR_DMA_SELECT;  in tegra_uart_hw_init()
1075 		tegra_uart_write(tup, tup->fcr_shadow, UART_FCR);  in tegra_uart_hw_init()
1077 		tegra_uart_write(tup, tup->fcr_shadow, UART_FCR);  in tegra_uart_hw_init()
1079 	tup->rx_in_progress = 1;  in tegra_uart_hw_init()
1095 	tup->ier_shadow = UART_IER_RLSI | UART_IER_RTOIE | UART_IER_RDI;  in tegra_uart_hw_init()
1101 	if (!tup->use_rx_pio)  in tegra_uart_hw_init()
1102 		tup->ier_shadow |= TEGRA_UART_IER_EORD;  in tegra_uart_hw_init()
1104 	tegra_uart_write(tup, tup->ier_shadow, UART_IER);  in tegra_uart_hw_init()
1108 static void tegra_uart_dma_channel_free(struct tegra_uart_port *tup,  in tegra_uart_dma_channel_free()  argument
1112 		dmaengine_terminate_all(tup->rx_dma_chan);  in tegra_uart_dma_channel_free()
1113 		dma_release_channel(tup->rx_dma_chan);  in tegra_uart_dma_channel_free()
1114 		dma_free_coherent(tup->uport.dev, TEGRA_UART_RX_DMA_BUFFER_SIZE,  in tegra_uart_dma_channel_free()
1115 				tup->rx_dma_buf_virt, tup->rx_dma_buf_phys);  in tegra_uart_dma_channel_free()
1116 		tup->rx_dma_chan = NULL;  in tegra_uart_dma_channel_free()
1117 		tup->rx_dma_buf_phys = 0;  in tegra_uart_dma_channel_free()
1118 		tup->rx_dma_buf_virt = NULL;  in tegra_uart_dma_channel_free()
1120 		dmaengine_terminate_all(tup->tx_dma_chan);  in tegra_uart_dma_channel_free()
1121 		dma_release_channel(tup->tx_dma_chan);  in tegra_uart_dma_channel_free()
1122 		dma_unmap_single(tup->uport.dev, tup->tx_dma_buf_phys,  in tegra_uart_dma_channel_free()
1124 		tup->tx_dma_chan = NULL;  in tegra_uart_dma_channel_free()
1125 		tup->tx_dma_buf_phys = 0;  in tegra_uart_dma_channel_free()
1126 		tup->tx_dma_buf_virt = NULL;  in tegra_uart_dma_channel_free()
1130 static int tegra_uart_dma_channel_allocate(struct tegra_uart_port *tup,  in tegra_uart_dma_channel_allocate()  argument
1139 	dma_chan = dma_request_chan(tup->uport.dev, dma_to_memory ? "rx" : "tx");  in tegra_uart_dma_channel_allocate()
1142 		dev_err(tup->uport.dev,  in tegra_uart_dma_channel_allocate()
1148 		dma_buf = dma_alloc_coherent(tup->uport.dev,  in tegra_uart_dma_channel_allocate()
1152 			dev_err(tup->uport.dev,  in tegra_uart_dma_channel_allocate()
1157 		dma_sync_single_for_device(tup->uport.dev, dma_phys,  in tegra_uart_dma_channel_allocate()
1160 		dma_sconfig.src_addr = tup->uport.mapbase;  in tegra_uart_dma_channel_allocate()
1162 		dma_sconfig.src_maxburst = tup->cdata->max_dma_burst_bytes;  in tegra_uart_dma_channel_allocate()
1163 		tup->rx_dma_chan = dma_chan;  in tegra_uart_dma_channel_allocate()
1164 		tup->rx_dma_buf_virt = dma_buf;  in tegra_uart_dma_channel_allocate()
1165 		tup->rx_dma_buf_phys = dma_phys;  in tegra_uart_dma_channel_allocate()
1167 		dma_phys = dma_map_single(tup->uport.dev,  in tegra_uart_dma_channel_allocate()
1168 			tup->uport.state->xmit.buf, UART_XMIT_SIZE,  in tegra_uart_dma_channel_allocate()
1170 		if (dma_mapping_error(tup->uport.dev, dma_phys)) {  in tegra_uart_dma_channel_allocate()
1171 			dev_err(tup->uport.dev, "dma_map_single tx failed\n");  in tegra_uart_dma_channel_allocate()
1175 		dma_buf = tup->uport.state->xmit.buf;  in tegra_uart_dma_channel_allocate()
1176 		dma_sconfig.dst_addr = tup->uport.mapbase;  in tegra_uart_dma_channel_allocate()
1179 		tup->tx_dma_chan = dma_chan;  in tegra_uart_dma_channel_allocate()
1180 		tup->tx_dma_buf_virt = dma_buf;  in tegra_uart_dma_channel_allocate()
1181 		tup->tx_dma_buf_phys = dma_phys;  in tegra_uart_dma_channel_allocate()
1186 		dev_err(tup->uport.dev,  in tegra_uart_dma_channel_allocate()
1188 		tegra_uart_dma_channel_free(tup, dma_to_memory);  in tegra_uart_dma_channel_allocate()
1197 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_startup()  local
1200 	if (!tup->use_tx_pio) {  in tegra_uart_startup()
1201 		ret = tegra_uart_dma_channel_allocate(tup, false);  in tegra_uart_startup()
1209 	if (!tup->use_rx_pio) {  in tegra_uart_startup()
1210 		ret = tegra_uart_dma_channel_allocate(tup, true);  in tegra_uart_startup()
1218 	ret = tegra_uart_hw_init(tup);  in tegra_uart_startup()
1225 				dev_name(u->dev), tup);  in tegra_uart_startup()
1233 	if (!tup->use_rx_pio)  in tegra_uart_startup()
1234 		tegra_uart_dma_channel_free(tup, true);  in tegra_uart_startup()
1236 	if (!tup->use_tx_pio)  in tegra_uart_startup()
1237 		tegra_uart_dma_channel_free(tup, false);  in tegra_uart_startup()
1247 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_flush_buffer()  local
1249 	tup->tx_bytes = 0;  in tegra_uart_flush_buffer()
1250 	if (tup->tx_dma_chan)  in tegra_uart_flush_buffer()
1251 		dmaengine_terminate_all(tup->tx_dma_chan);  in tegra_uart_flush_buffer()
1256 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_shutdown()  local
1258 	tegra_uart_hw_deinit(tup);  in tegra_uart_shutdown()
1259 	free_irq(u->irq, tup);  in tegra_uart_shutdown()
1264 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_enable_ms()  local
1266 	if (tup->enable_modem_interrupt) {  in tegra_uart_enable_ms()
1267 		tup->ier_shadow |= UART_IER_MSI;  in tegra_uart_enable_ms()
1268 		tegra_uart_write(tup, tup->ier_shadow, UART_IER);  in tegra_uart_enable_ms()
1276 	struct tegra_uart_port *tup = to_tegra_uport(u);  in tegra_uart_set_termios()  local
1281 	struct clk *parent_clk = clk_get_parent(tup->uart_clk);  in tegra_uart_set_termios()
1283 	int max_divider = (tup->cdata->support_clk_src_div) ? 0x7FFF : 0xFFFF;  in tegra_uart_set_termios()
1290 	if (tup->rts_active)  in tegra_uart_set_termios()
1291 		set_rts(tup, false);  in tegra_uart_set_termios()
1294 	tegra_uart_write(tup, tup->ier_shadow | UART_IER_RDI, UART_IER);  in tegra_uart_set_termios()
1295 	tegra_uart_read(tup, UART_IER);  in tegra_uart_set_termios()
1296 	tegra_uart_write(tup, 0, UART_IER);  in tegra_uart_set_termios()
1297 	tegra_uart_read(tup, UART_IER);  in tegra_uart_set_termios()
1300 	lcr = tup->lcr_shadow;  in tegra_uart_set_termios()
1328 	tegra_uart_write(tup, lcr, UART_LCR);  in tegra_uart_set_termios()
1329 	tup->lcr_shadow = lcr;  in tegra_uart_set_termios()
1330 	tup->symb_bit = tty_get_frame_size(termios->c_cflag);  in tegra_uart_set_termios()
1337 	ret = tegra_set_baudrate(tup, baud);  in tegra_uart_set_termios()
1339 		dev_err(tup->uport.dev, "Failed to set baud rate\n");  in tegra_uart_set_termios()
1348 		tup->mcr_shadow |= TEGRA_UART_MCR_CTS_EN;  in tegra_uart_set_termios()
1349 		tup->mcr_shadow &= ~TEGRA_UART_MCR_RTS_EN;  in tegra_uart_set_termios()
1350 		tegra_uart_write(tup, tup->mcr_shadow, UART_MCR);  in tegra_uart_set_termios()
1352 		if (tup->rts_active)  in tegra_uart_set_termios()
1353 			set_rts(tup, true);  in tegra_uart_set_termios()
1355 		tup->mcr_shadow &= ~TEGRA_UART_MCR_CTS_EN;  in tegra_uart_set_termios()
1356 		tup->mcr_shadow &= ~TEGRA_UART_MCR_RTS_EN;  in tegra_uart_set_termios()
1357 		tegra_uart_write(tup, tup->mcr_shadow, UART_MCR);  in tegra_uart_set_termios()
1364 	tegra_uart_read(tup, UART_IER);  in tegra_uart_set_termios()
1367 	tegra_uart_write(tup, tup->ier_shadow, UART_IER);  in tegra_uart_set_termios()
1368 	tegra_uart_read(tup, UART_IER);  in tegra_uart_set_termios()
1370 	tup->uport.ignore_status_mask = 0;  in tegra_uart_set_termios()
1373 		tup->uport.ignore_status_mask |= UART_LSR_DR;  in tegra_uart_set_termios()
1375 		tup->uport.ignore_status_mask |= UART_LSR_BI;  in tegra_uart_set_termios()
1412 	struct tegra_uart_port *tup)  in tegra_uart_parse_dt()  argument
1427 	tup->uport.line = port;  in tegra_uart_parse_dt()
1429 	tup->enable_modem_interrupt = of_property_read_bool(np,  in tegra_uart_parse_dt()
1434 		tup->use_rx_pio = true;  in tegra_uart_parse_dt()
1439 		tup->use_tx_pio = true;  in tegra_uart_parse_dt()
1445 		tup->n_adjustable_baud_rates = n_entries / 3;  in tegra_uart_parse_dt()
1446 		tup->baud_tolerance =  in tegra_uart_parse_dt()
1447 		devm_kzalloc(&pdev->dev, (tup->n_adjustable_baud_rates) *  in tegra_uart_parse_dt()
1448 			     sizeof(*tup->baud_tolerance), GFP_KERNEL);  in tegra_uart_parse_dt()
1449 		if (!tup->baud_tolerance)  in tegra_uart_parse_dt()
1458 				tup->baud_tolerance[index].lower_range_baud =  in tegra_uart_parse_dt()
1465 				tup->baud_tolerance[index].upper_range_baud =  in tegra_uart_parse_dt()
1472 				tup->baud_tolerance[index].tolerance =  in tegra_uart_parse_dt()
1476 		tup->n_adjustable_baud_rates = 0;  in tegra_uart_parse_dt()
1546 	struct tegra_uart_port *tup;  in tegra_uart_probe()  local
1558 	tup = devm_kzalloc(&pdev->dev, sizeof(*tup), GFP_KERNEL);  in tegra_uart_probe()
1559 	if (!tup) {  in tegra_uart_probe()
1564 	ret = tegra_uart_parse_dt(pdev, tup);  in tegra_uart_probe()
1568 	u = &tup->uport;  in tegra_uart_probe()
1573 	tup->cdata = cdata;  in tegra_uart_probe()
1575 	platform_set_drvdata(pdev, tup);  in tegra_uart_probe()
1587 	tup->uart_clk = devm_clk_get(&pdev->dev, NULL);  in tegra_uart_probe()
1588 	if (IS_ERR(tup->uart_clk)) {  in tegra_uart_probe()
1590 		return PTR_ERR(tup->uart_clk);  in tegra_uart_probe()
1593 	tup->rst = devm_reset_control_get_exclusive(&pdev->dev, "serial");  in tegra_uart_probe()
1594 	if (IS_ERR(tup->rst)) {  in tegra_uart_probe()
1596 		return PTR_ERR(tup->rst);  in tegra_uart_probe()
1615 	struct tegra_uart_port *tup = platform_get_drvdata(pdev);  in tegra_uart_remove()  local
1616 	struct uart_port *u = &tup->uport;  in tegra_uart_remove()
1625 	struct tegra_uart_port *tup = dev_get_drvdata(dev);  in tegra_uart_suspend()  local
1626 	struct uart_port *u = &tup->uport;  in tegra_uart_suspend()
1633 	struct tegra_uart_port *tup = dev_get_drvdata(dev);  in tegra_uart_resume()  local
1634 	struct uart_port *u = &tup->uport;  in tegra_uart_resume()