Lines Matching +full:diff +full:- +full:channels
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
41 "Maximum number of channels to use (default: all)");
51 "dmatest 0-memcpy 1-memset (default: 0)");
66 "Pass -1 for infinite timeout");
80 static int alignment = -1;
82 …M_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
93 * struct dmatest_params - test parameters.
98 * @max_channels: maximum number of channels to use
102 * @timeout: transfer timeout in msec, -1 for infinite timeout
127 * struct dmatest_info - test information.
129 * @channels: channels under test
130 * @nr_channels: number of channels under test
140 struct list_head channels; member
146 .channels = LIST_HEAD_INIT(test_info.channels),
210 /* poor man's completion - we want to use wait_event_freezable() on it */
250 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_run()
253 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_run()
254 if (!thread->done && !thread->pending) in is_threaded_test_run()
266 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_pending()
269 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_pending()
270 if (thread->pending) in is_threaded_test_pending()
281 struct dmatest_params *params = &info->params; in dmatest_wait_get()
283 if (params->iterations) in dmatest_wait_get()
299 if (params->channel[0] == '\0') in dmatest_match_channel()
301 return strcmp(dma_chan_name(chan), params->channel) == 0; in dmatest_match_channel()
307 if (params->device[0] == '\0') in dmatest_match_device()
309 return strcmp(dev_name(device->dev), params->device) == 0; in dmatest_match_device()
374 u8 diff = actual ^ pattern; in dmatest_mismatch() local
376 const char *thread_name = current->comm; in dmatest_mismatch()
382 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) in dmatest_mismatch()
385 else if (diff & PATTERN_SRC) in dmatest_mismatch()
422 current->comm, error_count - MAX_ERROR_COUNT); in dmatest_verify()
433 if (!thread->done) { in dmatest_callback()
434 done->done = true; in dmatest_callback()
435 wake_up_all(done->wait); in dmatest_callback()
438 * If thread->done, it means that this callback occurred in dmatest_callback()
452 return val % 2 ? val : val - 1; in min_odd()
460 current->comm, n, err, src_off, dst_off, len, data); in result()
463 current->comm, n, err, src_off, dst_off, len, data); in result()
472 current->comm, n, err, src_off, dst_off, len, data); in dbg_result()
489 /* drop precision until runtime is 32-bits */ in dmatest_persec()
512 kfree(d->raw[i]); in __dmatest_free_test_data()
514 kfree(d->aligned); in __dmatest_free_test_data()
515 kfree(d->raw); in __dmatest_free_test_data()
520 __dmatest_free_test_data(d, d->cnt); in dmatest_free_test_data()
528 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
529 if (!d->raw) in dmatest_alloc_test_data()
530 return -ENOMEM; in dmatest_alloc_test_data()
532 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
533 if (!d->aligned) in dmatest_alloc_test_data()
536 for (i = 0; i < d->cnt; i++) { in dmatest_alloc_test_data()
537 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); in dmatest_alloc_test_data()
538 if (!d->raw[i]) in dmatest_alloc_test_data()
543 d->aligned[i] = PTR_ALIGN(d->raw[i], align); in dmatest_alloc_test_data()
545 d->aligned[i] = d->raw[i]; in dmatest_alloc_test_data()
551 return -ENOMEM; in dmatest_alloc_test_data()
558 * in parallel for a single channel, and there may be multiple channels
571 struct dmatest_done *done = &thread->test_done; in dmatest_func()
589 ktime_t ktime, start, diff; in dmatest_func() local
602 ret = -ENOMEM; in dmatest_func()
605 thread->pending = false; in dmatest_func()
606 info = thread->info; in dmatest_func()
607 params = &info->params; in dmatest_func()
608 chan = thread->chan; in dmatest_func()
609 dev = chan->device; in dmatest_func()
612 src = &thread->src; in dmatest_func()
613 dst = &thread->dst; in dmatest_func()
614 if (thread->type == DMA_MEMCPY) { in dmatest_func()
615 align = params->alignment < 0 ? dev->copy_align : in dmatest_func()
616 params->alignment; in dmatest_func()
617 src->cnt = dst->cnt = 1; in dmatest_func()
618 } else if (thread->type == DMA_MEMSET) { in dmatest_func()
619 align = params->alignment < 0 ? dev->fill_align : in dmatest_func()
620 params->alignment; in dmatest_func()
621 src->cnt = dst->cnt = 1; in dmatest_func()
623 } else if (thread->type == DMA_XOR) { in dmatest_func()
625 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor); in dmatest_func()
626 dst->cnt = 1; in dmatest_func()
627 align = params->alignment < 0 ? dev->xor_align : in dmatest_func()
628 params->alignment; in dmatest_func()
629 } else if (thread->type == DMA_PQ) { in dmatest_func()
631 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); in dmatest_func()
632 dst->cnt = 2; in dmatest_func()
633 align = params->alignment < 0 ? dev->pq_align : in dmatest_func()
634 params->alignment; in dmatest_func()
636 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); in dmatest_func()
640 for (i = 0; i < src->cnt; i++) in dmatest_func()
646 if ((src->cnt + dst->cnt) >= 255) { in dmatest_func()
648 src->cnt + dst->cnt); in dmatest_func()
652 buf_size = params->buf_size; in dmatest_func()
654 pr_err("%u-byte buffer too small for %d-byte alignment\n", in dmatest_func()
667 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
671 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
678 if (params->polled) in dmatest_func()
685 (params->iterations && total_tests >= params->iterations))) { in dmatest_func()
693 if (params->transfer_size) { in dmatest_func()
694 if (params->transfer_size >= buf_size) { in dmatest_func()
695 pr_err("%u-byte transfer size must be lower than %u-buffer size\n", in dmatest_func()
696 params->transfer_size, buf_size); in dmatest_func()
699 len = params->transfer_size; in dmatest_func()
700 } else if (params->norandom) { in dmatest_func()
707 if (!params->transfer_size) { in dmatest_func()
714 if (params->norandom) { in dmatest_func()
715 src->off = 0; in dmatest_func()
716 dst->off = 0; in dmatest_func()
718 src->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
719 dst->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
721 src->off = (src->off >> align) << align; in dmatest_func()
722 dst->off = (dst->off >> align) << align; in dmatest_func()
725 if (!params->noverify) { in dmatest_func()
727 dmatest_init_srcs(src->aligned, src->off, len, in dmatest_func()
729 dmatest_init_dsts(dst->aligned, dst->off, len, in dmatest_func()
732 diff = ktime_sub(ktime_get(), start); in dmatest_func()
733 filltime = ktime_add(filltime, diff); in dmatest_func()
736 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt, in dmatest_func()
741 src->off, dst->off, len, ret); in dmatest_func()
745 um->len = buf_size; in dmatest_func()
746 for (i = 0; i < src->cnt; i++) { in dmatest_func()
747 void *buf = src->aligned[i]; in dmatest_func()
751 um->addr[i] = dma_map_page(dma_dev, pg, pg_off, in dmatest_func()
752 um->len, DMA_TO_DEVICE); in dmatest_func()
753 srcs[i] = um->addr[i] + src->off; in dmatest_func()
754 ret = dma_mapping_error(dma_dev, um->addr[i]); in dmatest_func()
757 src->off, dst->off, len, ret); in dmatest_func()
760 um->to_cnt++; in dmatest_func()
763 dsts = &um->addr[src->cnt]; in dmatest_func()
764 for (i = 0; i < dst->cnt; i++) { in dmatest_func()
765 void *buf = dst->aligned[i]; in dmatest_func()
769 dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len, in dmatest_func()
774 src->off, dst->off, len, ret); in dmatest_func()
777 um->bidi_cnt++; in dmatest_func()
780 if (thread->type == DMA_MEMCPY) in dmatest_func()
781 tx = dev->device_prep_dma_memcpy(chan, in dmatest_func()
782 dsts[0] + dst->off, in dmatest_func()
784 else if (thread->type == DMA_MEMSET) in dmatest_func()
785 tx = dev->device_prep_dma_memset(chan, in dmatest_func()
786 dsts[0] + dst->off, in dmatest_func()
787 *(src->aligned[0] + src->off), in dmatest_func()
789 else if (thread->type == DMA_XOR) in dmatest_func()
790 tx = dev->device_prep_dma_xor(chan, in dmatest_func()
791 dsts[0] + dst->off, in dmatest_func()
792 srcs, src->cnt, in dmatest_func()
794 else if (thread->type == DMA_PQ) { in dmatest_func()
795 for (i = 0; i < dst->cnt; i++) in dmatest_func()
796 dma_pq[i] = dsts[i] + dst->off; in dmatest_func()
797 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, in dmatest_func()
798 src->cnt, pq_coefs, in dmatest_func()
803 result("prep error", total_tests, src->off, in dmatest_func()
804 dst->off, len, ret); in dmatest_func()
809 done->done = false; in dmatest_func()
810 if (!params->polled) { in dmatest_func()
811 tx->callback = dmatest_callback; in dmatest_func()
812 tx->callback_param = done; in dmatest_func()
814 cookie = tx->tx_submit(tx); in dmatest_func()
817 result("submit error", total_tests, src->off, in dmatest_func()
818 dst->off, len, ret); in dmatest_func()
823 if (params->polled) { in dmatest_func()
827 done->done = true; in dmatest_func()
831 wait_event_freezable_timeout(thread->done_wait, in dmatest_func()
832 done->done, in dmatest_func()
833 msecs_to_jiffies(params->timeout)); in dmatest_func()
839 if (!done->done) { in dmatest_func()
840 result("test timed out", total_tests, src->off, dst->off, in dmatest_func()
845 dev->cap_mask) && in dmatest_func()
849 "completion busy status", total_tests, src->off, in dmatest_func()
850 dst->off, len, ret); in dmatest_func()
856 if (params->noverify) { in dmatest_func()
857 verbose_result("test passed", total_tests, src->off, in dmatest_func()
858 dst->off, len, 0); in dmatest_func()
863 pr_debug("%s: verifying source buffer...\n", current->comm); in dmatest_func()
864 error_count = dmatest_verify(src->aligned, 0, src->off, in dmatest_func()
866 error_count += dmatest_verify(src->aligned, src->off, in dmatest_func()
867 src->off + len, src->off, in dmatest_func()
869 error_count += dmatest_verify(src->aligned, src->off + len, in dmatest_func()
870 buf_size, src->off + len, in dmatest_func()
873 pr_debug("%s: verifying dest buffer...\n", current->comm); in dmatest_func()
874 error_count += dmatest_verify(dst->aligned, 0, dst->off, in dmatest_func()
877 error_count += dmatest_verify(dst->aligned, dst->off, in dmatest_func()
878 dst->off + len, src->off, in dmatest_func()
881 error_count += dmatest_verify(dst->aligned, dst->off + len, in dmatest_func()
882 buf_size, dst->off + len, in dmatest_func()
885 diff = ktime_sub(ktime_get(), start); in dmatest_func()
886 comparetime = ktime_add(comparetime, diff); in dmatest_func()
889 result("data error", total_tests, src->off, dst->off, in dmatest_func()
893 verbose_result("test passed", total_tests, src->off, in dmatest_func()
894 dst->off, len, 0); in dmatest_func()
921 current->comm, total_tests, failed_tests, in dmatest_func()
925 /* terminate all transfers on specified channels */ in dmatest_func()
929 thread->done = true; in dmatest_func()
941 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { in dmatest_cleanup_channel()
942 ret = kthread_stop(thread->task); in dmatest_cleanup_channel()
944 thread->task->comm, ret); in dmatest_cleanup_channel()
945 list_del(&thread->node); in dmatest_cleanup_channel()
946 put_task_struct(thread->task); in dmatest_cleanup_channel()
950 /* terminate all transfers on specified channels */ in dmatest_cleanup_channel()
951 dmaengine_terminate_sync(dtc->chan); in dmatest_cleanup_channel()
959 struct dmatest_params *params = &info->params; in dmatest_add_threads()
961 struct dma_chan *chan = dtc->chan; in dmatest_add_threads()
974 return -EINVAL; in dmatest_add_threads()
976 for (i = 0; i < params->threads_per_chan; i++) { in dmatest_add_threads()
979 pr_warn("No memory for %s-%s%u\n", in dmatest_add_threads()
983 thread->info = info; in dmatest_add_threads()
984 thread->chan = dtc->chan; in dmatest_add_threads()
985 thread->type = type; in dmatest_add_threads()
986 thread->test_done.wait = &thread->done_wait; in dmatest_add_threads()
987 init_waitqueue_head(&thread->done_wait); in dmatest_add_threads()
989 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", in dmatest_add_threads()
991 if (IS_ERR(thread->task)) { in dmatest_add_threads()
992 pr_warn("Failed to create thread %s-%s%u\n", in dmatest_add_threads()
999 get_task_struct(thread->task); in dmatest_add_threads()
1000 list_add_tail(&thread->node, &dtc->threads); in dmatest_add_threads()
1001 thread->pending = true; in dmatest_add_threads()
1011 struct dma_device *dma_dev = chan->device; in dmatest_add_channel()
1018 return -ENOMEM; in dmatest_add_channel()
1021 dtc->chan = chan; in dmatest_add_channel()
1022 INIT_LIST_HEAD(&dtc->threads); in dmatest_add_channel()
1024 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) && in dmatest_add_channel()
1025 info->params.polled) { in dmatest_add_channel()
1026 info->params.polled = false; in dmatest_add_channel()
1030 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { in dmatest_add_channel()
1037 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { in dmatest_add_channel()
1044 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { in dmatest_add_channel()
1048 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { in dmatest_add_channel()
1056 list_add_tail(&dtc->node, &info->channels); in dmatest_add_channel()
1057 info->nr_channels++; in dmatest_add_channel()
1064 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device); in filter()
1075 struct dmatest_params *params = &info->params; in request_channels()
1085 break; /* no more channels available */ in request_channels()
1086 if (params->max_channels && in request_channels()
1087 info->nr_channels >= params->max_channels) in request_channels()
1094 struct dmatest_params *params = &info->params; in add_threaded_test()
1097 params->buf_size = test_buf_size; in add_threaded_test()
1098 strlcpy(params->channel, strim(test_channel), sizeof(params->channel)); in add_threaded_test()
1099 strlcpy(params->device, strim(test_device), sizeof(params->device)); in add_threaded_test()
1100 params->threads_per_chan = threads_per_chan; in add_threaded_test()
1101 params->max_channels = max_channels; in add_threaded_test()
1102 params->iterations = iterations; in add_threaded_test()
1103 params->xor_sources = xor_sources; in add_threaded_test()
1104 params->pq_sources = pq_sources; in add_threaded_test()
1105 params->timeout = timeout; in add_threaded_test()
1106 params->noverify = noverify; in add_threaded_test()
1107 params->norandom = norandom; in add_threaded_test()
1108 params->alignment = alignment; in add_threaded_test()
1109 params->transfer_size = transfer_size; in add_threaded_test()
1110 params->polled = polled; in add_threaded_test()
1123 list_for_each_entry(dtc, &info->channels, node) { in run_pending_tests()
1127 list_for_each_entry(thread, &dtc->threads, node) { in run_pending_tests()
1128 wake_up_process(thread->task); in run_pending_tests()
1132 thread_count, dma_chan_name(dtc->chan)); in run_pending_tests()
1141 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { in stop_threaded_test()
1142 list_del(&dtc->node); in stop_threaded_test()
1143 chan = dtc->chan; in stop_threaded_test()
1149 info->nr_channels = 0; in stop_threaded_test()
1157 if (!info->did_init) in start_threaded_tests()
1167 mutex_lock(&info->lock); in dmatest_run_get()
1175 mutex_unlock(&info->lock); in dmatest_run_get()
1185 mutex_lock(&info->lock); in dmatest_run_set()
1188 mutex_unlock(&info->lock); in dmatest_run_set()
1195 ret = info->last_error; in dmatest_run_set()
1199 mutex_unlock(&info->lock); in dmatest_run_set()
1203 pr_info("No channels configured, continue with any\n"); in dmatest_run_set()
1214 mutex_unlock(&info->lock); in dmatest_run_set()
1226 mutex_lock(&info->lock); in dmatest_chan_set()
1229 mutex_unlock(&info->lock); in dmatest_chan_set()
1235 /* Reject channels that are already registered */ in dmatest_chan_set()
1237 list_for_each_entry(dtc, &info->channels, node) { in dmatest_chan_set()
1238 if (strcmp(dma_chan_name(dtc->chan), in dmatest_chan_set()
1240 dtc = list_last_entry(&info->channels, in dmatest_chan_set()
1244 dma_chan_name(dtc->chan), in dmatest_chan_set()
1246 ret = -EBUSY; in dmatest_chan_set()
1255 if (!list_empty(&info->channels)) { in dmatest_chan_set()
1262 dtc = list_last_entry(&info->channels, struct dmatest_chan, node); in dmatest_chan_set()
1263 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0) in dmatest_chan_set()
1265 ret = -EINVAL; in dmatest_chan_set()
1266 strlcpy(chan_reset_val, dma_chan_name(dtc->chan), in dmatest_chan_set()
1272 /* Clear test_channel if no channels were added successfully */ in dmatest_chan_set()
1274 ret = -EBUSY; in dmatest_chan_set()
1278 info->last_error = ret; in dmatest_chan_set()
1279 mutex_unlock(&info->lock); in dmatest_chan_set()
1285 info->last_error = ret; in dmatest_chan_set()
1286 mutex_unlock(&info->lock); in dmatest_chan_set()
1295 mutex_lock(&info->lock); in dmatest_chan_get()
1300 mutex_unlock(&info->lock); in dmatest_chan_get()
1311 list_for_each_entry(dtc, &info->channels, node) { in dmatest_test_list_get()
1315 list_for_each_entry(thread, &dtc->threads, node) { in dmatest_test_list_get()
1319 thread_count, dma_chan_name(dtc->chan)); in dmatest_test_list_get()
1328 struct dmatest_params *params = &info->params; in dmatest_init()
1331 mutex_lock(&info->lock); in dmatest_init()
1334 mutex_unlock(&info->lock); in dmatest_init()
1337 if (params->iterations && wait) in dmatest_init()
1343 info->did_init = true; in dmatest_init()
1347 /* when compiled-in wait for drivers to load first */
1354 mutex_lock(&info->lock); in dmatest_exit()
1356 mutex_unlock(&info->lock); in dmatest_exit()