772 	struct perf_thread *pthr = data;  in perf_dma_copy_callback()  local
774 	atomic_dec(&pthr->dma_sync);  in perf_dma_copy_callback()
775 	wake_up(&pthr->dma_wait);  in perf_dma_copy_callback()
778 static int perf_copy_chunk(struct perf_thread *pthr,  in perf_copy_chunk()  argument
791 	dma_dev = pthr->dma_chan->device->dev;  in perf_copy_chunk()
793 	if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),  in perf_copy_chunk()
819 		tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],  in perf_copy_chunk()
831 	tx->callback_param = pthr;  in perf_copy_chunk()
842 	atomic_inc(&pthr->dma_sync);  in perf_copy_chunk()
843 	dma_async_issue_pending(pthr->dma_chan);  in perf_copy_chunk()
846 	return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;  in perf_copy_chunk()
864 static int perf_init_test(struct perf_thread *pthr)  in perf_init_test()  argument
866 	struct perf_ctx *perf = pthr->perf;  in perf_init_test()
869 	pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,  in perf_init_test()
871 	if (!pthr->src)  in perf_init_test()
874 	get_random_bytes(pthr->src, perf->test_peer->outbuf_size);  in perf_init_test()
881 	pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);  in perf_init_test()
882 	if (!pthr->dma_chan) {  in perf_init_test()
884 			pthr->tidx);  in perf_init_test()
887 		kfree(pthr->src);  in perf_init_test()
891 	atomic_set(&pthr->dma_sync, 0);  in perf_init_test()
896 static int perf_run_test(struct perf_thread *pthr)  in perf_run_test()  argument
898 	struct perf_peer *peer = pthr->perf->test_peer;  in perf_run_test()
899 	struct perf_ctx *perf = pthr->perf;  in perf_run_test()
909 	flt_src = pthr->src;  in perf_run_test()
913 	pthr->duration = ktime_get();  in perf_run_test()
916 	while (pthr->copied < total_size) {  in perf_run_test()
917 		ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);  in perf_run_test()
920 				pthr->tidx, ret);  in perf_run_test()
924 		pthr->copied += chunk_size;  in perf_run_test()
930 			flt_src = pthr->src;  in perf_run_test()
940 static int perf_sync_test(struct perf_thread *pthr)  in perf_sync_test()  argument
942 	struct perf_ctx *perf = pthr->perf;  in perf_sync_test()
947 	wait_event(pthr->dma_wait,  in perf_sync_test()
948 		   (atomic_read(&pthr->dma_sync) == 0 ||  in perf_sync_test()
955 	pthr->duration = ktime_sub(ktime_get(), pthr->duration);  in perf_sync_test()
958 		pthr->tidx, pthr->copied);  in perf_sync_test()
961 		pthr->tidx, ktime_to_us(pthr->duration));  in perf_sync_test()
963 	dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,  in perf_sync_test()
964 		div64_u64(pthr->copied, ktime_to_us(pthr->duration)));  in perf_sync_test()
969 static void perf_clear_test(struct perf_thread *pthr)  in perf_clear_test()  argument
971 	struct perf_ctx *perf = pthr->perf;  in perf_clear_test()
980 	(void)dmaengine_terminate_sync(pthr->dma_chan);  in perf_clear_test()
982 	dma_release_channel(pthr->dma_chan);  in perf_clear_test()
987 	kfree(pthr->src);  in perf_clear_test()
992 	struct perf_thread *pthr = to_thread_work(work);  in perf_thread_work()  local
1002 	ret = perf_init_test(pthr);  in perf_thread_work()
1004 		pthr->status = ret;  in perf_thread_work()
1008 	ret = perf_run_test(pthr);  in perf_thread_work()
1010 		pthr->status = ret;  in perf_thread_work()
1014 	pthr->status = perf_sync_test(pthr);  in perf_thread_work()
1017 	perf_clear_test(pthr);  in perf_thread_work()
1051 	struct perf_thread *pthr;  in perf_submit_test()  local
1064 		pthr = &perf->threads[tidx];  in perf_submit_test()
1066 		pthr->status = -ENODATA;  in perf_submit_test()
1067 		pthr->copied = 0;  in perf_submit_test()
1068 		pthr->duration = ktime_set(0, 0);  in perf_submit_test()
1070 			(void)queue_work(perf_wq, &pthr->work);  in perf_submit_test()
1088 	struct perf_thread *pthr;  in perf_read_stats()  local
1098 		pthr = &perf->threads[tidx];  in perf_read_stats()
1100 		if (pthr->status == -ENODATA)  in perf_read_stats()
1103 		if (pthr->status) {  in perf_read_stats()
1105 				"%d: error status %d\n", tidx, pthr->status);  in perf_read_stats()
1111 			tidx, pthr->copied, ktime_to_us(pthr->duration),  in perf_read_stats()
1112 			div64_u64(pthr->copied, ktime_to_us(pthr->duration)));  in perf_read_stats()
1122 	struct perf_thread *pthr;  in perf_init_threads()  local
1130 		pthr = &perf->threads[tidx];  in perf_init_threads()
1132 		pthr->perf = perf;  in perf_init_threads()
1133 		pthr->tidx = tidx;  in perf_init_threads()
1134 		pthr->status = -ENODATA;  in perf_init_threads()
1135 		init_waitqueue_head(&pthr->dma_wait);  in perf_init_threads()
1136 		INIT_WORK(&pthr->work, perf_thread_work);  in perf_init_threads()