Lines Matching +full:diff +full:- +full:channels
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <linux/dma-mapping.h>
40 "Maximum number of channels to use (default: all)");
50 "dmatest 0-memcpy 1-memset (default: 0)");
65 "Pass -1 for infinite timeout");
79 static int alignment = -1;
81 …M_DESC(alignment, "Custom data address alignment taken as 2^(alignment) (default: not used (-1))");
92 * struct dmatest_params - test parameters.
97 * @max_channels: maximum number of channels to use
101 * @timeout: transfer timeout in msec, -1 for infinite timeout
126 * struct dmatest_info - test information.
128 * @channels: channels under test
129 * @nr_channels: number of channels under test
139 struct list_head channels; member
145 .channels = LIST_HEAD_INIT(test_info.channels),
209 /* poor man's completion - we want to use wait_event_freezable() on it */
249 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_run()
252 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_run()
253 if (!thread->done && !thread->pending) in is_threaded_test_run()
265 list_for_each_entry(dtc, &info->channels, node) { in is_threaded_test_pending()
268 list_for_each_entry(thread, &dtc->threads, node) { in is_threaded_test_pending()
269 if (thread->pending) in is_threaded_test_pending()
280 struct dmatest_params *params = &info->params; in dmatest_wait_get()
282 if (params->iterations) in dmatest_wait_get()
298 if (params->channel[0] == '\0') in dmatest_match_channel()
300 return strcmp(dma_chan_name(chan), params->channel) == 0; in dmatest_match_channel()
306 if (params->device[0] == '\0') in dmatest_match_device()
308 return strcmp(dev_name(device->dev), params->device) == 0; in dmatest_match_device()
373 u8 diff = actual ^ pattern; in dmatest_mismatch() local
375 const char *thread_name = current->comm; in dmatest_mismatch()
381 && (diff & (PATTERN_COPY | PATTERN_OVERWRITE))) in dmatest_mismatch()
384 else if (diff & PATTERN_SRC) in dmatest_mismatch()
421 current->comm, error_count - MAX_ERROR_COUNT); in dmatest_verify()
432 if (!thread->done) { in dmatest_callback()
433 done->done = true; in dmatest_callback()
434 wake_up_all(done->wait); in dmatest_callback()
437 * If thread->done, it means that this callback occurred in dmatest_callback()
451 return val % 2 ? val : val - 1; in min_odd()
459 current->comm, n, err, src_off, dst_off, len, data); in result()
462 current->comm, n, err, src_off, dst_off, len, data); in result()
471 current->comm, n, err, src_off, dst_off, len, data); in dbg_result()
488 /* drop precision until runtime is 32-bits */ in dmatest_persec()
511 kfree(d->raw[i]); in __dmatest_free_test_data()
513 kfree(d->aligned); in __dmatest_free_test_data()
514 kfree(d->raw); in __dmatest_free_test_data()
519 __dmatest_free_test_data(d, d->cnt); in dmatest_free_test_data()
527 d->raw = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
528 if (!d->raw) in dmatest_alloc_test_data()
529 return -ENOMEM; in dmatest_alloc_test_data()
531 d->aligned = kcalloc(d->cnt + 1, sizeof(u8 *), GFP_KERNEL); in dmatest_alloc_test_data()
532 if (!d->aligned) in dmatest_alloc_test_data()
535 for (i = 0; i < d->cnt; i++) { in dmatest_alloc_test_data()
536 d->raw[i] = kmalloc(buf_size + align, GFP_KERNEL); in dmatest_alloc_test_data()
537 if (!d->raw[i]) in dmatest_alloc_test_data()
542 d->aligned[i] = PTR_ALIGN(d->raw[i], align); in dmatest_alloc_test_data()
544 d->aligned[i] = d->raw[i]; in dmatest_alloc_test_data()
550 return -ENOMEM; in dmatest_alloc_test_data()
557 * in parallel for a single channel, and there may be multiple channels
570 struct dmatest_done *done = &thread->test_done; in dmatest_func()
588 ktime_t ktime, start, diff; in dmatest_func() local
601 ret = -ENOMEM; in dmatest_func()
604 thread->pending = false; in dmatest_func()
605 info = thread->info; in dmatest_func()
606 params = &info->params; in dmatest_func()
607 chan = thread->chan; in dmatest_func()
608 dev = chan->device; in dmatest_func()
611 src = &thread->src; in dmatest_func()
612 dst = &thread->dst; in dmatest_func()
613 if (thread->type == DMA_MEMCPY) { in dmatest_func()
614 align = params->alignment < 0 ? dev->copy_align : in dmatest_func()
615 params->alignment; in dmatest_func()
616 src->cnt = dst->cnt = 1; in dmatest_func()
617 } else if (thread->type == DMA_MEMSET) { in dmatest_func()
618 align = params->alignment < 0 ? dev->fill_align : in dmatest_func()
619 params->alignment; in dmatest_func()
620 src->cnt = dst->cnt = 1; in dmatest_func()
622 } else if (thread->type == DMA_XOR) { in dmatest_func()
624 src->cnt = min_odd(params->xor_sources | 1, dev->max_xor); in dmatest_func()
625 dst->cnt = 1; in dmatest_func()
626 align = params->alignment < 0 ? dev->xor_align : in dmatest_func()
627 params->alignment; in dmatest_func()
628 } else if (thread->type == DMA_PQ) { in dmatest_func()
630 src->cnt = min_odd(params->pq_sources | 1, dma_maxpq(dev, 0)); in dmatest_func()
631 dst->cnt = 2; in dmatest_func()
632 align = params->alignment < 0 ? dev->pq_align : in dmatest_func()
633 params->alignment; in dmatest_func()
635 pq_coefs = kmalloc(params->pq_sources + 1, GFP_KERNEL); in dmatest_func()
639 for (i = 0; i < src->cnt; i++) in dmatest_func()
645 if ((src->cnt + dst->cnt) >= 255) { in dmatest_func()
647 src->cnt + dst->cnt); in dmatest_func()
651 buf_size = params->buf_size; in dmatest_func()
653 pr_err("%u-byte buffer too small for %d-byte alignment\n", in dmatest_func()
666 srcs = kcalloc(src->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
670 dma_pq = kcalloc(dst->cnt, sizeof(dma_addr_t), GFP_KERNEL); in dmatest_func()
677 if (params->polled) in dmatest_func()
684 (params->iterations && total_tests >= params->iterations))) { in dmatest_func()
692 if (params->transfer_size) { in dmatest_func()
693 if (params->transfer_size >= buf_size) { in dmatest_func()
694 pr_err("%u-byte transfer size must be lower than %u-buffer size\n", in dmatest_func()
695 params->transfer_size, buf_size); in dmatest_func()
698 len = params->transfer_size; in dmatest_func()
699 } else if (params->norandom) { in dmatest_func()
706 if (!params->transfer_size) { in dmatest_func()
713 if (params->norandom) { in dmatest_func()
714 src->off = 0; in dmatest_func()
715 dst->off = 0; in dmatest_func()
717 src->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
718 dst->off = dmatest_random() % (buf_size - len + 1); in dmatest_func()
720 src->off = (src->off >> align) << align; in dmatest_func()
721 dst->off = (dst->off >> align) << align; in dmatest_func()
724 if (!params->noverify) { in dmatest_func()
726 dmatest_init_srcs(src->aligned, src->off, len, in dmatest_func()
728 dmatest_init_dsts(dst->aligned, dst->off, len, in dmatest_func()
731 diff = ktime_sub(ktime_get(), start); in dmatest_func()
732 filltime = ktime_add(filltime, diff); in dmatest_func()
735 um = dmaengine_get_unmap_data(dma_dev, src->cnt + dst->cnt, in dmatest_func()
740 src->off, dst->off, len, ret); in dmatest_func()
744 um->len = buf_size; in dmatest_func()
745 for (i = 0; i < src->cnt; i++) { in dmatest_func()
746 void *buf = src->aligned[i]; in dmatest_func()
750 um->addr[i] = dma_map_page(dma_dev, pg, pg_off, in dmatest_func()
751 um->len, DMA_TO_DEVICE); in dmatest_func()
752 srcs[i] = um->addr[i] + src->off; in dmatest_func()
753 ret = dma_mapping_error(dma_dev, um->addr[i]); in dmatest_func()
756 src->off, dst->off, len, ret); in dmatest_func()
759 um->to_cnt++; in dmatest_func()
762 dsts = &um->addr[src->cnt]; in dmatest_func()
763 for (i = 0; i < dst->cnt; i++) { in dmatest_func()
764 void *buf = dst->aligned[i]; in dmatest_func()
768 dsts[i] = dma_map_page(dma_dev, pg, pg_off, um->len, in dmatest_func()
773 src->off, dst->off, len, ret); in dmatest_func()
776 um->bidi_cnt++; in dmatest_func()
779 if (thread->type == DMA_MEMCPY) in dmatest_func()
780 tx = dev->device_prep_dma_memcpy(chan, in dmatest_func()
781 dsts[0] + dst->off, in dmatest_func()
783 else if (thread->type == DMA_MEMSET) in dmatest_func()
784 tx = dev->device_prep_dma_memset(chan, in dmatest_func()
785 dsts[0] + dst->off, in dmatest_func()
786 *(src->aligned[0] + src->off), in dmatest_func()
788 else if (thread->type == DMA_XOR) in dmatest_func()
789 tx = dev->device_prep_dma_xor(chan, in dmatest_func()
790 dsts[0] + dst->off, in dmatest_func()
791 srcs, src->cnt, in dmatest_func()
793 else if (thread->type == DMA_PQ) { in dmatest_func()
794 for (i = 0; i < dst->cnt; i++) in dmatest_func()
795 dma_pq[i] = dsts[i] + dst->off; in dmatest_func()
796 tx = dev->device_prep_dma_pq(chan, dma_pq, srcs, in dmatest_func()
797 src->cnt, pq_coefs, in dmatest_func()
802 result("prep error", total_tests, src->off, in dmatest_func()
803 dst->off, len, ret); in dmatest_func()
808 done->done = false; in dmatest_func()
809 if (!params->polled) { in dmatest_func()
810 tx->callback = dmatest_callback; in dmatest_func()
811 tx->callback_param = done; in dmatest_func()
813 cookie = tx->tx_submit(tx); in dmatest_func()
816 result("submit error", total_tests, src->off, in dmatest_func()
817 dst->off, len, ret); in dmatest_func()
822 if (params->polled) { in dmatest_func()
826 done->done = true; in dmatest_func()
830 wait_event_freezable_timeout(thread->done_wait, in dmatest_func()
831 done->done, in dmatest_func()
832 msecs_to_jiffies(params->timeout)); in dmatest_func()
838 if (!done->done) { in dmatest_func()
839 result("test timed out", total_tests, src->off, dst->off, in dmatest_func()
844 dev->cap_mask) && in dmatest_func()
848 "completion busy status", total_tests, src->off, in dmatest_func()
849 dst->off, len, ret); in dmatest_func()
855 if (params->noverify) { in dmatest_func()
856 verbose_result("test passed", total_tests, src->off, in dmatest_func()
857 dst->off, len, 0); in dmatest_func()
862 pr_debug("%s: verifying source buffer...\n", current->comm); in dmatest_func()
863 error_count = dmatest_verify(src->aligned, 0, src->off, in dmatest_func()
865 error_count += dmatest_verify(src->aligned, src->off, in dmatest_func()
866 src->off + len, src->off, in dmatest_func()
868 error_count += dmatest_verify(src->aligned, src->off + len, in dmatest_func()
869 buf_size, src->off + len, in dmatest_func()
872 pr_debug("%s: verifying dest buffer...\n", current->comm); in dmatest_func()
873 error_count += dmatest_verify(dst->aligned, 0, dst->off, in dmatest_func()
876 error_count += dmatest_verify(dst->aligned, dst->off, in dmatest_func()
877 dst->off + len, src->off, in dmatest_func()
880 error_count += dmatest_verify(dst->aligned, dst->off + len, in dmatest_func()
881 buf_size, dst->off + len, in dmatest_func()
884 diff = ktime_sub(ktime_get(), start); in dmatest_func()
885 comparetime = ktime_add(comparetime, diff); in dmatest_func()
888 result("data error", total_tests, src->off, dst->off, in dmatest_func()
892 verbose_result("test passed", total_tests, src->off, in dmatest_func()
893 dst->off, len, 0); in dmatest_func()
920 current->comm, total_tests, failed_tests, in dmatest_func()
924 /* terminate all transfers on specified channels */ in dmatest_func()
928 thread->done = true; in dmatest_func()
940 list_for_each_entry_safe(thread, _thread, &dtc->threads, node) { in dmatest_cleanup_channel()
941 ret = kthread_stop(thread->task); in dmatest_cleanup_channel()
943 thread->task->comm, ret); in dmatest_cleanup_channel()
944 list_del(&thread->node); in dmatest_cleanup_channel()
945 put_task_struct(thread->task); in dmatest_cleanup_channel()
949 /* terminate all transfers on specified channels */ in dmatest_cleanup_channel()
950 dmaengine_terminate_sync(dtc->chan); in dmatest_cleanup_channel()
958 struct dmatest_params *params = &info->params; in dmatest_add_threads()
960 struct dma_chan *chan = dtc->chan; in dmatest_add_threads()
973 return -EINVAL; in dmatest_add_threads()
975 for (i = 0; i < params->threads_per_chan; i++) { in dmatest_add_threads()
978 pr_warn("No memory for %s-%s%u\n", in dmatest_add_threads()
982 thread->info = info; in dmatest_add_threads()
983 thread->chan = dtc->chan; in dmatest_add_threads()
984 thread->type = type; in dmatest_add_threads()
985 thread->test_done.wait = &thread->done_wait; in dmatest_add_threads()
986 init_waitqueue_head(&thread->done_wait); in dmatest_add_threads()
988 thread->task = kthread_create(dmatest_func, thread, "%s-%s%u", in dmatest_add_threads()
990 if (IS_ERR(thread->task)) { in dmatest_add_threads()
991 pr_warn("Failed to create thread %s-%s%u\n", in dmatest_add_threads()
998 get_task_struct(thread->task); in dmatest_add_threads()
999 list_add_tail(&thread->node, &dtc->threads); in dmatest_add_threads()
1000 thread->pending = true; in dmatest_add_threads()
1010 struct dma_device *dma_dev = chan->device; in dmatest_add_channel()
1017 return -ENOMEM; in dmatest_add_channel()
1020 dtc->chan = chan; in dmatest_add_channel()
1021 INIT_LIST_HEAD(&dtc->threads); in dmatest_add_channel()
1023 if (dma_has_cap(DMA_COMPLETION_NO_ORDER, dma_dev->cap_mask) && in dmatest_add_channel()
1024 info->params.polled) { in dmatest_add_channel()
1025 info->params.polled = false; in dmatest_add_channel()
1029 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) { in dmatest_add_channel()
1036 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) { in dmatest_add_channel()
1043 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { in dmatest_add_channel()
1047 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { in dmatest_add_channel()
1055 list_add_tail(&dtc->node, &info->channels); in dmatest_add_channel()
1056 info->nr_channels++; in dmatest_add_channel()
1063 return dmatest_match_channel(param, chan) && dmatest_match_device(param, chan->device); in filter()
1074 struct dmatest_params *params = &info->params; in request_channels()
1084 break; /* no more channels available */ in request_channels()
1085 if (params->max_channels && in request_channels()
1086 info->nr_channels >= params->max_channels) in request_channels()
1093 struct dmatest_params *params = &info->params; in add_threaded_test()
1096 params->buf_size = test_buf_size; in add_threaded_test()
1097 strscpy(params->channel, strim(test_channel), sizeof(params->channel)); in add_threaded_test()
1098 strscpy(params->device, strim(test_device), sizeof(params->device)); in add_threaded_test()
1099 params->threads_per_chan = threads_per_chan; in add_threaded_test()
1100 params->max_channels = max_channels; in add_threaded_test()
1101 params->iterations = iterations; in add_threaded_test()
1102 params->xor_sources = xor_sources; in add_threaded_test()
1103 params->pq_sources = pq_sources; in add_threaded_test()
1104 params->timeout = timeout; in add_threaded_test()
1105 params->noverify = noverify; in add_threaded_test()
1106 params->norandom = norandom; in add_threaded_test()
1107 params->alignment = alignment; in add_threaded_test()
1108 params->transfer_size = transfer_size; in add_threaded_test()
1109 params->polled = polled; in add_threaded_test()
1122 list_for_each_entry(dtc, &info->channels, node) { in run_pending_tests()
1126 list_for_each_entry(thread, &dtc->threads, node) { in run_pending_tests()
1127 wake_up_process(thread->task); in run_pending_tests()
1131 thread_count, dma_chan_name(dtc->chan)); in run_pending_tests()
1140 list_for_each_entry_safe(dtc, _dtc, &info->channels, node) { in stop_threaded_test()
1141 list_del(&dtc->node); in stop_threaded_test()
1142 chan = dtc->chan; in stop_threaded_test()
1148 info->nr_channels = 0; in stop_threaded_test()
1156 if (!info->did_init) in start_threaded_tests()
1166 mutex_lock(&info->lock); in dmatest_run_get()
1174 mutex_unlock(&info->lock); in dmatest_run_get()
1184 mutex_lock(&info->lock); in dmatest_run_set()
1187 mutex_unlock(&info->lock); in dmatest_run_set()
1194 ret = info->last_error; in dmatest_run_set()
1198 mutex_unlock(&info->lock); in dmatest_run_set()
1202 pr_info("No channels configured, continue with any\n"); in dmatest_run_set()
1213 mutex_unlock(&info->lock); in dmatest_run_set()
1225 mutex_lock(&info->lock); in dmatest_chan_set()
1228 mutex_unlock(&info->lock); in dmatest_chan_set()
1234 /* Reject channels that are already registered */ in dmatest_chan_set()
1236 list_for_each_entry(dtc, &info->channels, node) { in dmatest_chan_set()
1237 if (strcmp(dma_chan_name(dtc->chan), in dmatest_chan_set()
1239 dtc = list_last_entry(&info->channels, in dmatest_chan_set()
1243 dma_chan_name(dtc->chan), in dmatest_chan_set()
1245 ret = -EBUSY; in dmatest_chan_set()
1254 if (!list_empty(&info->channels)) { in dmatest_chan_set()
1261 dtc = list_last_entry(&info->channels, struct dmatest_chan, node); in dmatest_chan_set()
1262 if ((strcmp(dma_chan_name(dtc->chan), strim(test_channel)) != 0) in dmatest_chan_set()
1264 ret = -EINVAL; in dmatest_chan_set()
1265 strscpy(chan_reset_val, dma_chan_name(dtc->chan), in dmatest_chan_set()
1271 /* Clear test_channel if no channels were added successfully */ in dmatest_chan_set()
1273 ret = -EBUSY; in dmatest_chan_set()
1277 info->last_error = ret; in dmatest_chan_set()
1278 mutex_unlock(&info->lock); in dmatest_chan_set()
1284 info->last_error = ret; in dmatest_chan_set()
1285 mutex_unlock(&info->lock); in dmatest_chan_set()
1294 mutex_lock(&info->lock); in dmatest_chan_get()
1299 mutex_unlock(&info->lock); in dmatest_chan_get()
1310 list_for_each_entry(dtc, &info->channels, node) { in dmatest_test_list_get()
1314 list_for_each_entry(thread, &dtc->threads, node) { in dmatest_test_list_get()
1318 thread_count, dma_chan_name(dtc->chan)); in dmatest_test_list_get()
1327 struct dmatest_params *params = &info->params; in dmatest_init()
1330 mutex_lock(&info->lock); in dmatest_init()
1333 mutex_unlock(&info->lock); in dmatest_init()
1336 if (params->iterations && wait) in dmatest_init()
1342 info->did_init = true; in dmatest_init()
1346 /* when compiled-in wait for drivers to load first */
1353 mutex_lock(&info->lock); in dmatest_exit()
1355 mutex_unlock(&info->lock); in dmatest_exit()