289 				       struct spi_transfer *xfer,  in spi_statistics_add_transfer_stats()  argument
293 	int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1;  in spi_statistics_add_transfer_stats()
303 	stats->bytes += xfer->len;  in spi_statistics_add_transfer_stats()
304 	if ((xfer->tx_buf) &&  in spi_statistics_add_transfer_stats()
305 	    (xfer->tx_buf != ctlr->dummy_tx))  in spi_statistics_add_transfer_stats()
306 		stats->bytes_tx += xfer->len;  in spi_statistics_add_transfer_stats()
307 	if ((xfer->rx_buf) &&  in spi_statistics_add_transfer_stats()
308 	    (xfer->rx_buf != ctlr->dummy_rx))  in spi_statistics_add_transfer_stats()
309 		stats->bytes_rx += xfer->len;  in spi_statistics_add_transfer_stats()
976 	struct spi_transfer *xfer;  in __spi_map_msg()  local
996 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_map_msg()
997 		if (!ctlr->can_dma(ctlr, msg->spi, xfer))  in __spi_map_msg()
1000 		if (xfer->tx_buf != NULL) {  in __spi_map_msg()
1001 			ret = spi_map_buf(ctlr, tx_dev, &xfer->tx_sg,  in __spi_map_msg()
1002 					  (void *)xfer->tx_buf, xfer->len,  in __spi_map_msg()
1008 		if (xfer->rx_buf != NULL) {  in __spi_map_msg()
1009 			ret = spi_map_buf(ctlr, rx_dev, &xfer->rx_sg,  in __spi_map_msg()
1010 					  xfer->rx_buf, xfer->len,  in __spi_map_msg()
1013 				spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg,  in __spi_map_msg()
1027 	struct spi_transfer *xfer;  in __spi_unmap_msg()  local
1043 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_unmap_msg()
1044 		if (!ctlr->can_dma(ctlr, msg->spi, xfer))  in __spi_unmap_msg()
1047 		spi_unmap_buf(ctlr, rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);  in __spi_unmap_msg()
1048 		spi_unmap_buf(ctlr, tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);  in __spi_unmap_msg()
1072 	struct spi_transfer *xfer;  in spi_unmap_msg()  local
1074 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_unmap_msg()
1079 		if (xfer->tx_buf == ctlr->dummy_tx)  in spi_unmap_msg()
1080 			xfer->tx_buf = NULL;  in spi_unmap_msg()
1081 		if (xfer->rx_buf == ctlr->dummy_rx)  in spi_unmap_msg()
1082 			xfer->rx_buf = NULL;  in spi_unmap_msg()
1090 	struct spi_transfer *xfer;  in spi_map_msg()  local
1099 		list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_map_msg()
1101 			    !xfer->tx_buf)  in spi_map_msg()
1102 				max_tx = max(xfer->len, max_tx);  in spi_map_msg()
1104 			    !xfer->rx_buf)  in spi_map_msg()
1105 				max_rx = max(xfer->len, max_rx);  in spi_map_msg()
1126 			list_for_each_entry(xfer, &msg->transfers,  in spi_map_msg()
1128 				if (!xfer->len)  in spi_map_msg()
1130 				if (!xfer->tx_buf)  in spi_map_msg()
1131 					xfer->tx_buf = ctlr->dummy_tx;  in spi_map_msg()
1132 				if (!xfer->rx_buf)  in spi_map_msg()
1133 					xfer->rx_buf = ctlr->dummy_rx;  in spi_map_msg()
1143 			     struct spi_transfer *xfer)  in spi_transfer_wait()  argument
1147 	u32 speed_hz = xfer->speed_hz;  in spi_transfer_wait()
1165 		ms = 8LL * MSEC_PER_SEC * xfer->len;  in spi_transfer_wait()
1207 int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer)  in spi_delay_to_ns()  argument
1225 		if (!xfer)  in spi_delay_to_ns()
1231 		hz = xfer->effective_speed_hz ?: xfer->speed_hz / 2;  in spi_delay_to_ns()
1246 int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer)  in spi_delay_exec()  argument
1255 	delay = spi_delay_to_ns(_delay, xfer);  in spi_delay_exec()
1266 					  struct spi_transfer *xfer)  in _spi_transfer_cs_change_delay()  argument
1269 	u32 delay = xfer->cs_change_delay.value;  in _spi_transfer_cs_change_delay()
1270 	u32 unit = xfer->cs_change_delay.unit;  in _spi_transfer_cs_change_delay()
1280 	ret = spi_delay_exec(&xfer->cs_change_delay, xfer);  in _spi_transfer_cs_change_delay()
1299 	struct spi_transfer *xfer;  in spi_transfer_one_message()  local
1310 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_transfer_one_message()
1311 		trace_spi_transfer_start(msg, xfer);  in spi_transfer_one_message()
1313 		spi_statistics_add_transfer_stats(statm, xfer, ctlr);  in spi_transfer_one_message()
1314 		spi_statistics_add_transfer_stats(stats, xfer, ctlr);  in spi_transfer_one_message()
1317 			xfer->ptp_sts_word_pre = 0;  in spi_transfer_one_message()
1318 			ptp_read_system_prets(xfer->ptp_sts);  in spi_transfer_one_message()
1321 		if ((xfer->tx_buf || xfer->rx_buf) && xfer->len) {  in spi_transfer_one_message()
1325 			ret = ctlr->transfer_one(ctlr, msg->spi, xfer);  in spi_transfer_one_message()
1328 				   (xfer->error & SPI_TRANS_FAIL_NO_START)) {  in spi_transfer_one_message()
1331 					xfer->error &= ~SPI_TRANS_FAIL_NO_START;  in spi_transfer_one_message()
1345 				ret = spi_transfer_wait(ctlr, msg, xfer);  in spi_transfer_one_message()
1350 			if (xfer->len)  in spi_transfer_one_message()
1353 					xfer->len);  in spi_transfer_one_message()
1357 			ptp_read_system_postts(xfer->ptp_sts);  in spi_transfer_one_message()
1358 			xfer->ptp_sts_word_post = xfer->len;  in spi_transfer_one_message()
1361 		trace_spi_transfer_stop(msg, xfer);  in spi_transfer_one_message()
1366 		spi_transfer_delay_exec(xfer);  in spi_transfer_one_message()
1368 		if (xfer->cs_change) {  in spi_transfer_one_message()
1369 			if (list_is_last(&xfer->transfer_list,  in spi_transfer_one_message()
1374 				_spi_transfer_cs_change_delay(msg, xfer);  in spi_transfer_one_message()
1379 		msg->actual_length += xfer->len;  in spi_transfer_one_message()
1434 	struct spi_transfer *xfer;  in __spi_pump_messages()  local
1566 		list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in __spi_pump_messages()
1567 			xfer->ptp_sts_word_pre = 0;  in __spi_pump_messages()
1568 			ptp_read_system_prets(xfer->ptp_sts);  in __spi_pump_messages()
1621 			    struct spi_transfer *xfer,  in spi_take_timestamp_pre()  argument
1624 	if (!xfer->ptp_sts)  in spi_take_timestamp_pre()
1627 	if (xfer->timestamped)  in spi_take_timestamp_pre()
1630 	if (progress > xfer->ptp_sts_word_pre)  in spi_take_timestamp_pre()
1634 	xfer->ptp_sts_word_pre = progress;  in spi_take_timestamp_pre()
1641 	ptp_read_system_prets(xfer->ptp_sts);  in spi_take_timestamp_pre()
1658 			     struct spi_transfer *xfer,  in spi_take_timestamp_post()  argument
1661 	if (!xfer->ptp_sts)  in spi_take_timestamp_post()
1664 	if (xfer->timestamped)  in spi_take_timestamp_post()
1667 	if (progress < xfer->ptp_sts_word_post)  in spi_take_timestamp_post()
1670 	ptp_read_system_postts(xfer->ptp_sts);  in spi_take_timestamp_post()
1678 	xfer->ptp_sts_word_post = progress;  in spi_take_timestamp_post()
1680 	xfer->timestamped = true;  in spi_take_timestamp_post()
1766 	struct spi_transfer *xfer;  in spi_finalize_current_message()  local
1776 		list_for_each_entry(xfer, &mesg->transfers, transfer_list) {  in spi_finalize_current_message()
1777 			ptp_read_system_postts(xfer->ptp_sts);  in spi_finalize_current_message()
1778 			xfer->ptp_sts_word_post = xfer->len;  in spi_finalize_current_message()
1783 		list_for_each_entry(xfer, &mesg->transfers, transfer_list)  in spi_finalize_current_message()
1784 			WARN_ON_ONCE(xfer->ptp_sts && !xfer->timestamped);  in spi_finalize_current_message()
3199 	struct spi_transfer *xfer;  in spi_replace_transfers()  local
3258 		xfer = &rxfer->inserted_transfers[insert - 1 - i];  in spi_replace_transfers()
3261 		memcpy(xfer, xfer_first, sizeof(*xfer));  in spi_replace_transfers()
3264 		list_add(&xfer->transfer_list, rxfer->replaced_after);  in spi_replace_transfers()
3268 			xfer->cs_change = false;  in spi_replace_transfers()
3269 			xfer->delay.value = 0;  in spi_replace_transfers()
3289 	struct spi_transfer *xfer = *xferp, *xfers;  in __spi_split_transfer_maxsize()  local
3295 	count = DIV_ROUND_UP(xfer->len, maxsize);  in __spi_split_transfer_maxsize()
3298 	srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp);  in __spi_split_transfer_maxsize()
3316 	xfers[0].len = min_t(size_t, maxsize, xfer[0].len);  in __spi_split_transfer_maxsize()
3364 	struct spi_transfer *xfer;  in spi_split_transfers_maxsize()  local
3373 	list_for_each_entry(xfer, &msg->transfers, transfer_list) {  in spi_split_transfers_maxsize()
3374 		if (xfer->len > maxsize) {  in spi_split_transfers_maxsize()
3375 			ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,  in spi_split_transfers_maxsize()
3549 static int _spi_xfer_word_delay_update(struct spi_transfer *xfer,  in _spi_xfer_word_delay_update()  argument
3554 	delay1 = spi_delay_to_ns(&xfer->word_delay, xfer);  in _spi_xfer_word_delay_update()
3558 	delay2 = spi_delay_to_ns(&spi->word_delay, xfer);  in _spi_xfer_word_delay_update()
3563 		memcpy(&xfer->word_delay, &spi->word_delay,  in _spi_xfer_word_delay_update()
3564 		       sizeof(xfer->word_delay));  in _spi_xfer_word_delay_update()
3572 	struct spi_transfer *xfer;  in __spi_validate()  local
3600 		list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_validate()
3602 			if (list_is_last(&xfer->transfer_list, &message->transfers))  in __spi_validate()
3604 			xfer->cs_change = 1;  in __spi_validate()
3617 		list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_validate()
3618 			if (xfer->rx_buf && xfer->tx_buf)  in __spi_validate()
3620 			if ((flags & SPI_CONTROLLER_NO_TX) && xfer->tx_buf)  in __spi_validate()
3622 			if ((flags & SPI_CONTROLLER_NO_RX) && xfer->rx_buf)  in __spi_validate()
3636 	list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_validate()
3637 		xfer->effective_speed_hz = 0;  in __spi_validate()
3638 		message->frame_length += xfer->len;  in __spi_validate()
3639 		if (!xfer->bits_per_word)  in __spi_validate()
3640 			xfer->bits_per_word = spi->bits_per_word;  in __spi_validate()
3642 		if (!xfer->speed_hz)  in __spi_validate()
3643 			xfer->speed_hz = spi->max_speed_hz;  in __spi_validate()
3645 		if (ctlr->max_speed_hz && xfer->speed_hz > ctlr->max_speed_hz)  in __spi_validate()
3646 			xfer->speed_hz = ctlr->max_speed_hz;  in __spi_validate()
3648 		if (__spi_validate_bits_per_word(ctlr, xfer->bits_per_word))  in __spi_validate()
3655 		if (xfer->bits_per_word <= 8)  in __spi_validate()
3657 		else if (xfer->bits_per_word <= 16)  in __spi_validate()
3663 		if (xfer->len % w_size)  in __spi_validate()
3666 		if (xfer->speed_hz && ctlr->min_speed_hz &&  in __spi_validate()
3667 		    xfer->speed_hz < ctlr->min_speed_hz)  in __spi_validate()
3670 		if (xfer->tx_buf && !xfer->tx_nbits)  in __spi_validate()
3671 			xfer->tx_nbits = SPI_NBITS_SINGLE;  in __spi_validate()
3672 		if (xfer->rx_buf && !xfer->rx_nbits)  in __spi_validate()
3673 			xfer->rx_nbits = SPI_NBITS_SINGLE;  in __spi_validate()
3678 		if (xfer->tx_buf) {  in __spi_validate()
3681 			if (xfer->tx_nbits != SPI_NBITS_SINGLE &&  in __spi_validate()
3682 				xfer->tx_nbits != SPI_NBITS_DUAL &&  in __spi_validate()
3683 				xfer->tx_nbits != SPI_NBITS_QUAD)  in __spi_validate()
3685 			if ((xfer->tx_nbits == SPI_NBITS_DUAL) &&  in __spi_validate()
3688 			if ((xfer->tx_nbits == SPI_NBITS_QUAD) &&  in __spi_validate()
3693 		if (xfer->rx_buf) {  in __spi_validate()
3696 			if (xfer->rx_nbits != SPI_NBITS_SINGLE &&  in __spi_validate()
3697 				xfer->rx_nbits != SPI_NBITS_DUAL &&  in __spi_validate()
3698 				xfer->rx_nbits != SPI_NBITS_QUAD)  in __spi_validate()
3700 			if ((xfer->rx_nbits == SPI_NBITS_DUAL) &&  in __spi_validate()
3703 			if ((xfer->rx_nbits == SPI_NBITS_QUAD) &&  in __spi_validate()
3708 		if (_spi_xfer_word_delay_update(xfer, spi))  in __spi_validate()
3720 	struct spi_transfer *xfer;  in __spi_async()  local
3737 		list_for_each_entry(xfer, &message->transfers, transfer_list) {  in __spi_async()
3738 			xfer->ptp_sts_word_pre = 0;  in __spi_async()
3739 			ptp_read_system_prets(xfer->ptp_sts);  in __spi_async()