49  * through the rx.channel pointer. This delineates ownership of the cache
68 	} rx;  member
98 	u32 rx = READ_ONCE(header->rx.count);  in tegra_ivc_empty()  local
110 	if (tx - rx > ivc->num_frames)  in tegra_ivc_empty()
113 	return tx == rx;  in tegra_ivc_empty()
120 	u32 rx = READ_ONCE(header->rx.count);  in tegra_ivc_full()  local
126 	return tx - rx >= ivc->num_frames;  in tegra_ivc_full()
133 	u32 rx = READ_ONCE(header->rx.count);  in tegra_ivc_available()  local
141 	return tx - rx;  in tegra_ivc_available()
157 	WRITE_ONCE(ivc->rx.channel->rx.count,  in tegra_ivc_advance_rx()
158 		   READ_ONCE(ivc->rx.channel->rx.count) + 1);  in tegra_ivc_advance_rx()
160 	if (ivc->rx.position == ivc->num_frames - 1)  in tegra_ivc_advance_rx()
161 		ivc->rx.position = 0;  in tegra_ivc_advance_rx()
163 		ivc->rx.position++;  in tegra_ivc_advance_rx()
175 	 * asynchronous transition of rx.channel->state to  in tegra_ivc_check_read()
188 	if (!tegra_ivc_empty(ivc, ivc->rx.channel))  in tegra_ivc_check_read()
191 	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);  in tegra_ivc_check_read()
193 	if (tegra_ivc_empty(ivc, ivc->rx.channel))  in tegra_ivc_check_read()
201 	unsigned int offset = offsetof(struct tegra_ivc_header, rx.count);  in tegra_ivc_check_write()
266 /* directly peek at the next frame rx'ed */
279 	 * Order observation of ivc->rx.position potentially indicating new  in tegra_ivc_read_get_next_frame()
284 	tegra_ivc_invalidate_frame(ivc, ivc->rx.phys, ivc->rx.position, 0,  in tegra_ivc_read_get_next_frame()
287 	return tegra_ivc_frame_virt(ivc, ivc->rx.channel, ivc->rx.position);  in tegra_ivc_read_get_next_frame()
293 	unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);  in tegra_ivc_read_advance()  local
308 	tegra_ivc_flush(ivc, ivc->rx.phys + rx);  in tegra_ivc_read_advance()
311 	 * Ensure our write to ivc->rx.position occurs before our read from  in tegra_ivc_read_advance()
321 	tegra_ivc_invalidate(ivc, ivc->rx.phys + tx);  in tegra_ivc_read_advance()
323 	if (tegra_ivc_available(ivc, ivc->rx.channel) == ivc->num_frames - 1)  in tegra_ivc_read_advance()
347 	unsigned int rx = offsetof(struct tegra_ivc_header, rx.count);  in tegra_ivc_write_advance()  local
368 	 * ivc->rx.position.  in tegra_ivc_write_advance()
377 	tegra_ivc_invalidate(ivc, ivc->tx.phys + rx);  in tegra_ivc_write_advance()
422 	tegra_ivc_invalidate(ivc, ivc->rx.phys + offset);  in tegra_ivc_notified()
423 	state = READ_ONCE(ivc->rx.channel->tx.state);  in tegra_ivc_notified()
440 		ivc->rx.channel->rx.count = 0;  in tegra_ivc_notified()
443 		ivc->rx.position = 0;  in tegra_ivc_notified()
479 		ivc->rx.channel->rx.count = 0;  in tegra_ivc_notified()
482 		ivc->rx.position = 0;  in tegra_ivc_notified()
561 static int tegra_ivc_check_params(unsigned long rx, unsigned long tx,  in tegra_ivc_check_params()  argument
566 	BUILD_BUG_ON(!IS_ALIGNED(offsetof(struct tegra_ivc_header, rx.count),  in tegra_ivc_check_params()
585 	if (!IS_ALIGNED(rx, TEGRA_IVC_ALIGN)) {  in tegra_ivc_check_params()
586 		pr_err("IVC channel start not aligned: %#lx\n", rx);  in tegra_ivc_check_params()
595 	if (rx < tx) {  in tegra_ivc_check_params()
596 		if (rx + frame_size * num_frames > tx) {  in tegra_ivc_check_params()
598 			       rx, frame_size * num_frames, tx);  in tegra_ivc_check_params()
602 		if (tx + frame_size * num_frames > rx) {  in tegra_ivc_check_params()
604 			       tx, frame_size * num_frames, rx);  in tegra_ivc_check_params()
612 int tegra_ivc_init(struct tegra_ivc *ivc, struct device *peer, void *rx,  in tegra_ivc_init()  argument
631 	err = tegra_ivc_check_params((unsigned long)rx, (unsigned long)tx,  in tegra_ivc_init()
639 		ivc->rx.phys = dma_map_single(peer, rx, queue_size,  in tegra_ivc_init()
641 		if (dma_mapping_error(peer, ivc->rx.phys))  in tegra_ivc_init()
647 			dma_unmap_single(peer, ivc->rx.phys, queue_size,  in tegra_ivc_init()
652 		ivc->rx.phys = rx_phys;  in tegra_ivc_init()
656 	ivc->rx.channel = rx;  in tegra_ivc_init()
669 	ivc->rx.position = 0;  in tegra_ivc_init()
681 		dma_unmap_single(ivc->peer, ivc->rx.phys, size,  in tegra_ivc_cleanup()