#!/usr/bin/env python3 # # SPDX-FileCopyrightText: Copyright 2010-2024 Arm Limited and/or its affiliates # # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the License); you may # not use this file except in compliance with the License. # You may obtain a copy of the License at # # www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an AS IS BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import os os.environ["TF_USE_LEGACY_KERAS"]="1" import sys import argparse from packaging import version from test_settings import TestSettings from conv_settings import ConvSettings from pooling_settings import PoolingSettings from softmax_settings import SoftmaxSettings from svdf_settings import SVDFSettings from add_mul_settings import AddMulSettings from lstm_settings import LSTMSettings from fully_connected_settings import FullyConnectedSettings import tensorflow as tf def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Generate input and refererence output data for unittests." " It can regenerate all data, load all stored data or a combination of it.") parser.add_argument('--dataset', type=str, default=None, help="Name of generated test set.") parser.add_argument('--regenerate-weights', action='store_true', help="Regenerate and store new weights.") parser.add_argument('--regenerate-input', action='store_true', help="Regenerate and store new input.") parser.add_argument('--regenerate-biases', action='store_true', help="Regenerate and store new biases.") parser.add_argument('-a', '--regenerate-all', action='store_true', help="Regenerate and store all data.") parser.add_argument('-t', '--testtype', type=str, default=None, choices=[ 'conv', 'depthwise_conv', 'avgpool', 'maxpool', 'fully_connected', 'softmax', 'svdf', 'add', 'mul', 'lstm', 'transpose_conv' ], help='Type of test. There are the operators that have unit tests.') parser.add_argument('--run-all-testsets', action='store_true', help="Run the script for all existing test " "sets. Regenerate all, partially all or no input data (output may still change, depending on" " changes in script) depending on regenerate flags. If used together with the -t flag, only" " tests of that type will be run.") parser.add_argument('--schema-file', type=str, help="Path to schema file. This may be needed for some tests.") parser.add_argument('--interpreter', type=str, default='tensorflow', choices=['tensorflow', 'tflite_runtime', 'tflite_micro'], help="Use interpreter from tensorflow or tflite_runtime. See README for more info.") return parser.parse_args() def load_testdata_sets(regenerate_input, regenerate_weights, regenerate_biases, schema_file, interpreter) -> dict: """ Add all new testdata sets here """ testdata_sets = {} type_of_test = 'conv' dataset = 'basic' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=1, x_in=5, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=False, interpreter=interpreter) dataset = 'stride2pad1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=1, x_in=7, y_in=7, w_x=3, w_y=3, stride_x=2, stride_y=2, pad=True, interpreter=interpreter) dataset = 'kernel1x1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=19, out_ch=7, x_in=7, y_in=5, w_x=1, w_y=1, stride_x=1, stride_y=1, pad=False, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'kernel1x1_stride_x' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=9, out_ch=5, x_in=7, y_in=4, w_x=1, w_y=1, stride_x=3, stride_y=1, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'kernel1x1_stride_x_y' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=23, out_ch=15, randmin=0, x_in=7, y_in=6, w_x=1, w_y=1, stride_x=2, stride_y=2, pad=False, out_activation_min=-6, out_activation_max=127, batches=3, interpreter=interpreter) dataset = 'kernel1x1_stride_x_y_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=1, w_y=1, stride_x=2, stride_y=2, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'kernel1x1_stride_x_y_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=1, w_y=1, stride_x=3, stride_y=3, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'conv_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=1, x_in=10, y_in=49, w_x=4, w_y=10, stride_x=1, stride_y=2, pad=True, out_activation_min=-127, out_activation_max=127, interpreter=interpreter) dataset = 'conv_1_x_n_1' # left and right pad, no non-padded elements testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=3, x_in=2, y_in=1, w_x=3, w_y=1, stride_x=1, stride_y=1, pad=True, out_activation_min=-127, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'conv_1_x_n_2' # no pad testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=3, x_in=296, y_in=1, w_x=48, w_y=1, stride_x=2, stride_y=1, pad=False, out_activation_min=-111, out_activation_max=127, interpreter=interpreter) dataset = 'conv_1_x_n_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=1, x_in=296, y_in=1, w_x=48, w_y=1, stride_x=2, stride_y=1, pad=True, out_activation_min=-111, out_activation_max=127, interpreter=interpreter) dataset = 'conv_1_x_n_4' # 0 left pad, 1 right pad testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=4, x_in=16, y_in=1, w_x=3, w_y=1, stride_x=2, stride_y=1, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter) dataset = 'conv_1_x_n_5' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=1, x_in=17, y_in=1, w_x=3, w_y=1, stride_x=3, stride_y=1, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter) dataset = 'conv_1_x_n_6_generic' # right_pad_num + no_pad_num + left_pad_num != output_x testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=16, x_in=4, y_in=1, w_x=8, w_y=1, stride_x=4, stride_y=1, pad=True, out_activation_min=-125, out_activation_max=126, interpreter=interpreter) dataset = 'conv_1_x_n_7' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=20, out_ch=15, x_in=148, y_in=1, w_x=32, w_y=1, stride_x=1, stride_y=1, pad=True, out_activation_min=-127, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'conv_1_x_n_8' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=12, out_ch=10, x_in=148, y_in=1, w_x=48, w_y=1, stride_x=2, stride_y=1, pad=True, out_activation_min=-127, out_activation_max=127, batches=2, interpreter=interpreter) dataset = 'conv_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=4, x_in=6, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-101, out_activation_max=127, interpreter=interpreter) dataset = 'conv_4' # batches > 2 testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=3, x_in=5, y_in=5, w_x=2, w_y=3, stride_x=2, stride_y=2, pad=False, out_activation_min=-109, out_activation_max=127, batches=3, interpreter=interpreter) dataset = 'conv_5' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=128, out_ch=1, x_in=128, y_in=1, w_x=3, w_y=3, stride_x=4, stride_y=4, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter) dataset = 'conv_out_activation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, interpreter=interpreter) dataset = 'conv_dilation_golden' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, batches=2, out_ch=3, x_in=6, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=True, out_activation_min=-128, out_activation_max=127, dilation_x=3, dilation_y=2, interpreter=interpreter) dataset = 'conv_2x2_dilation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=10, y_in=10, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=False, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'conv_2x3_dilation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'conv_3x2_dilation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=3, dilation_y=2, interpreter=interpreter) dataset = 'conv_2x2_dilation_5x5_input' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=5, y_in=5, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'conv_3x3_dilation_5x5_input' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=9, y_in=11, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'int16xint8' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=55, out_ch=4, x_in=10, y_in=8, w_x=9, w_y=4, stride_x=1, stride_y=1, pad=False, generate_bias=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'int16xint8_spill' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=55, out_ch=9, x_in=10, y_in=8, w_x=9, w_y=4, stride_x=1, stride_y=1, pad=False, generate_bias=False, randmin=TestSettings.INT16_MAX - 100, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MAX - 100, out_activation_max=TestSettings.INT16_MAX, bias_min=TestSettings.INT64_MIN, bias_max=TestSettings.INT64_MAX, int16xint8=True, interpreter=interpreter, weights_min=100) dataset = 'int16xint8_spill2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=12, out_ch=9, x_in=10, y_in=8, w_x=9, w_y=4, stride_x=1, stride_y=1, pad=False, generate_bias=False, randmin=TestSettings.INT16_MAX - 100, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MAX - 100, out_activation_max=TestSettings.INT16_MAX, bias_min=TestSettings.INT64_MIN, bias_max=TestSettings.INT64_MAX, int16xint8=True, interpreter=interpreter, weights_min=100) dataset = 'requantize_s64' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, generate_bias=True, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, bias_min=-0x300, bias_max=TestSettings.INT32_MAX + 1000, interpreter=interpreter) dataset = 'int16xint8_dilation_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=32, y_in=32, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, bias_min=-0x300, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'int16xint8_dilation_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=4, x_in=7, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-13335, out_activation_max=32767, int16xint8=True, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'int16xint8_dilation_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=4, x_in=7, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-13335, out_activation_max=32767, int16xint8=True, dilation_x=2, interpreter=interpreter) dataset = 'int16xint8xint32_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=55, out_ch=4, x_in=10, y_in=8, w_x=9, w_y=4, stride_x=1, stride_y=1, pad=False, generate_bias=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, int16xint8_int32=True, interpreter=interpreter) dataset = 'int16xint8xint32_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=55, out_ch=9, x_in=10, y_in=8, w_x=9, w_y=4, stride_x=1, stride_y=1, pad=False, generate_bias=False, randmin=TestSettings.INT16_MAX - 100, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MAX - 100, out_activation_max=TestSettings.INT16_MAX, bias_min=TestSettings.INT64_MIN, bias_max=TestSettings.INT64_MAX, int16xint8=True, int16xint8_int32=True, interpreter=interpreter, weights_min=100) dataset = 'int16xint8xint32_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, generate_bias=True, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, int16xint8_int32=True, bias_min=-0x300, bias_max=TestSettings.INT32_MAX + 1000, interpreter=interpreter) dataset = 'int16xint8xint32_4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=32, y_in=32, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, int16xint8_int32=True, bias_min=-0x300, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'int16xint8xint32_5' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=4, x_in=7, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-13335, out_activation_max=32767, int16xint8=True, int16xint8_int32=True, dilation_x=2, dilation_y=2, interpreter=interpreter) dataset = 'int16xint8xint32_6' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=4, x_in=7, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-13335, out_activation_max=32767, int16xint8=True, int16xint8_int32=True, dilation_x=2, interpreter=interpreter) dataset = 'grouped_conv_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=6, groups=2, x_in=5, y_in=5, w_x=2, w_y=2, generate_bias=False, stride_x=1, stride_y=1, pad=False, batches=2, interpreter=interpreter) dataset = 'grouped_conv_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=2, groups=2, x_in=7, y_in=3, w_x=1, w_y=2, generate_bias=True, stride_x=1, stride_y=1, pad=False, interpreter=interpreter) dataset = 'grouped_conv_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=4, groups=2, x_in=3, y_in=2, w_x=3, w_y=2, generate_bias=True, stride_x=2, stride_y=2, pad=True, batches=2, interpreter=interpreter) dataset = 'grouped_conv_4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=6, groups=3, x_in=9, y_in=9, w_x=2, w_y=2, generate_bias=True, stride_x=1, stride_y=1, dilation_x=3, dilation_y=3, pad=True, interpreter=interpreter) dataset = 'basic_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=1, x_in=5, y_in=8, w_x=2, w_y=4, stride_x=1, stride_y=1, pad=False, interpreter=interpreter, int4_weights=True) dataset = 'basic_2_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=15, x_in=15, y_in=15, w_x=5, w_y=5, stride_x=1, stride_y=1, pad=False, interpreter=interpreter, int4_weights=True) dataset = 'stride2pad1_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=1, x_in=7, y_in=7, w_x=3, w_y=3, stride_x=2, stride_y=2, pad=True, interpreter=interpreter, int4_weights=True) dataset = 'kernel1x1_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=19, out_ch=7, x_in=7, y_in=5, w_x=1, w_y=1, stride_x=1, stride_y=1, pad=False, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter, int4_weights=True, generate_bias=False) dataset = 'kernel1x1_int4_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=60, out_ch=60, x_in=60, y_in=1, w_x=1, w_y=1, stride_x=1, stride_y=1, pad=False, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-126, out_activation_max=127, batches=1, interpreter=interpreter, int4_weights=True) dataset = 'kernel1x1_int4_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=18, out_ch=47, x_in=43, y_in=1, w_x=1, w_y=1, stride_x=1, stride_y=1, pad=False, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-126, out_activation_max=127, batches=1, interpreter=interpreter, int4_weights=True) dataset = 'kernel1x1_stride_x_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=9, out_ch=5, x_in=7, y_in=4, w_x=1, w_y=1, stride_x=3, stride_y=1, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter, int4_weights=True) dataset = 'kernel1x1_stride_x_y_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=23, out_ch=15, randmin=0, x_in=7, y_in=6, w_x=1, w_y=1, stride_x=2, stride_y=2, pad=False, out_activation_min=-6, out_activation_max=127, batches=3, interpreter=interpreter, int4_weights=True) dataset = 'kernel1x1_stride_x_y_1_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=1, w_y=1, stride_x=2, stride_y=2, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter, int4_weights=True, generate_bias=False) dataset = 'kernel1x1_stride_x_y_2_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=1, w_y=1, stride_x=3, stride_y=3, pad=False, out_activation_min=-126, out_activation_max=127, batches=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_3_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=1, x_in=10, y_in=49, w_x=4, w_y=10, stride_x=1, stride_y=2, pad=True, out_activation_min=-127, out_activation_max=127, interpreter=interpreter, int4_weights=True, generate_bias=False) dataset = 'conv_1_x_n_1_int4' # left and right pad, no non-padded elements testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=3, x_in=2, y_in=1, w_x=3, w_y=1, stride_x=1, stride_y=1, pad=True, out_activation_min=-127, out_activation_max=127, batches=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_1_x_n_2_int4' # no pad testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=3, x_in=296, y_in=1, w_x=48, w_y=1, stride_x=2, stride_y=1, pad=False, out_activation_min=-111, out_activation_max=127, interpreter=interpreter, int4_weights=True, generate_bias=False) dataset = 'conv_1_x_n_3_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=1, x_in=296, y_in=1, w_x=48, w_y=1, stride_x=2, stride_y=1, pad=True, out_activation_min=-111, out_activation_max=127, interpreter=interpreter, int4_weights=True) dataset = 'conv_1_x_n_4_int4' # 0 left pad, 1 right pad testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=4, x_in=16, y_in=1, w_x=3, w_y=1, stride_x=2, stride_y=1, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter, int4_weights=True) dataset = 'conv_1_x_n_5_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=1, x_in=17, y_in=1, w_x=3, w_y=1, stride_x=3, stride_y=1, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter, int4_weights=True) dataset = 'conv_2_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=4, x_in=6, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-101, out_activation_max=127, interpreter=interpreter, int4_weights=True) dataset = 'conv_4_int4' # batches > 2 testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=3, x_in=5, y_in=5, w_x=2, w_y=3, stride_x=2, stride_y=2, pad=False, out_activation_min=-109, out_activation_max=127, batches=3, interpreter=interpreter, int4_weights=True) dataset = 'conv_5_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=128, out_ch=1, x_in=128, y_in=1, w_x=3, w_y=3, stride_x=4, stride_y=4, pad=True, out_activation_min=-88, out_activation_max=127, interpreter=interpreter, int4_weights=True, generate_bias=False) dataset = 'conv_out_activation_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, interpreter=interpreter, int4_weights=True) dataset = 'conv_dilation_golden_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, batches=2, out_ch=3, x_in=6, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=True, out_activation_min=-128, out_activation_max=127, dilation_x=3, dilation_y=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_2x2_dilation_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=10, y_in=10, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=False, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_2x3_dilation_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_3x2_dilation_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=3, y_in=3, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=3, dilation_y=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_2x2_dilation_5x5_input_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=5, y_in=5, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter, int4_weights=True) dataset = 'conv_3x3_dilation_5x5_input_int4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=9, y_in=11, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=True, out_activation_min=-61, out_activation_max=107, dilation_x=2, dilation_y=2, interpreter=interpreter, int4_weights=True) type_of_test = 'transpose_conv' dataset = 'transpose_conv_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=32, batches=2, out_ch=3, x_in=9, y_in=9, w_x=6, w_y=6, generate_bias=True, stride_x=2, stride_y=2, pad=True, interpreter=interpreter) dataset = 'transpose_conv_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=15, batches=1, out_ch=4, x_in=12, y_in=12, w_x=3, w_y=3, generate_bias=False, stride_x=3, stride_y=1, pad=False, interpreter=interpreter) dataset = 'transpose_conv_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=7, batches=3, out_ch=5, x_in=1, y_in=7, w_x=5, w_y=1, generate_bias=True, stride_x=2, stride_y=5, pad=True, interpreter=interpreter) dataset = 'transpose_conv_4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=32, batches=3, out_ch=5, x_in=1, y_in=7, w_x=3, w_y=3, generate_bias=False, stride_x=3, stride_y=1, pad=False, interpreter=interpreter) type_of_test = 'depthwise_conv' dataset = 'depthwise_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=9, x_in=6, y_in=5, w_x=3, w_y=4, stride_x=2, stride_y=2, pad=True, out_activation_min=-73, out_activation_max=127, interpreter=interpreter) dataset = 'depthwise_kernel_3x3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=5, w_x=3, w_y=3, stride_x=2, stride_y=2, pad=True, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-104, out_activation_max=127, interpreter=interpreter) dataset = 'depthwise_kernel_3x3_null_bias' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=5, w_x=3, w_y=3, stride_x=2, stride_y=2, pad=True, generate_bias=False, out_activation_min=-104, out_activation_max=127, interpreter=interpreter) dataset = 'depthwise_eq_in_out_ch' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=250, out_ch=250, x_in=7, y_in=5, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=True, interpreter=interpreter) dataset = 'depthwise_sub_block' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=9, out_ch=9, x_in=7, y_in=5, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, interpreter=interpreter) dataset = 'depthwise_x_stride' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=9, out_ch=9, x_in=7, y_in=5, w_x=2, w_y=2, stride_x=2, stride_y=1, pad=False, interpreter=interpreter) dataset = 'depthwise_out_activation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=3, x_in=6, y_in=5, w_x=3, w_y=4, pad=False, out_activation_min=-45, out_activation_max=103, interpreter=interpreter) dataset = 'depthwise_mult_batches' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=3, x_in=3, y_in=5, w_x=2, w_y=4, stride_x=2, stride_y=2, pad=True, batches=2, interpreter=interpreter) dataset = 'depthwise_null_bias_0' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=2, x_in=4, y_in=5, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=True, generate_bias=False, batches=1, interpreter=interpreter) dataset = 'depthwise_null_bias_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=16, x_in=4, y_in=5, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=True, generate_bias=False, batches=1, interpreter=interpreter) dataset = 'depthwise_dilation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=9, x_in=6, y_in=5, w_x=3, w_y=4, stride_x=1, stride_y=1, pad=True, out_activation_min=-70, out_activation_max=127, dilation_x=2, dilation_y=3, interpreter=interpreter) dataset = 'dw_int16xint8' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=8, x_in=9, y_in=5, w_x=3, w_y=4, stride_x=3, stride_y=2, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-21111, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_dilation' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=8, x_in=9, y_in=5, w_x=4, w_y=4, stride_x=1, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-32700, dilation_x=3, dilation_y=2, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_mult4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=8, x_in=4, y_in=5, w_x=3, w_y=4, stride_x=3, stride_y=2, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-32767, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_fast' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-17000, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_fast_multiple_batches_uneven_buffers' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=5, y_in=5, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-17000, out_activation_max=32767, int16xint8=True, batches=3, interpreter=interpreter) dataset = 'dw_int16xint8_fast_multiple_batches_uneven_buffers_null_bias' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=3, w_y=2, stride_x=1, stride_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-17000, out_activation_max=32767, int16xint8=True, batches=3, generate_bias=False, interpreter=interpreter) dataset = 'dw_int16xint8_fast_test_bias' nbr_of_out_channels = 8 bias = [i for i in range(nbr_of_out_channels)] testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=nbr_of_out_channels, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-17000, out_activation_max=32767, int16xint8=True, generate_bias=bias, interpreter=interpreter) dataset = 'dw_int16xint8_fast_null_bias' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-17000, out_activation_max=32767, int16xint8=True, generate_bias=False, interpreter=interpreter) dataset = 'dw_int16xint8_fast_stride' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=2, stride_y=2, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, batches=2, out_activation_min=TestSettings.INT16_MIN, out_activation_max=16000, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_fast_stride_null_bias' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=8, out_ch=8, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=2, stride_y=2, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, batches=2, out_activation_min=TestSettings.INT16_MIN, out_activation_max=16000, int16xint8=True, generate_bias=False, interpreter=interpreter) dataset = 'dw_int16xint8_fast_spill' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=3, w_y=3, stride_x=2, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, batches=3, out_activation_min=-30000, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'dw_int16xint8_fast_spill_null_bias' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=5, out_ch=5, x_in=4, y_in=4, w_x=3, w_y=3, stride_x=2, stride_y=1, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, batches=3, out_activation_min=-30000, out_activation_max=32767, int16xint8=True, generate_bias=False, interpreter=interpreter) dataset = 'depthwise_int4_1' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=22, out_ch=22, x_in=1, y_in=23, w_x=1, w_y=3, stride_x=1, stride_y=1, pad=False, out_activation_min=-127, out_activation_max=127, generate_bias=False, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=19, out_ch=19, x_in=6, y_in=6, w_x=5, w_y=5, stride_x=1, stride_y=1, pad=False, out_activation_min=-127, out_activation_max=127, generate_bias=False, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=1, x_in=2, y_in=2, w_x=2, w_y=2, stride_x=1, stride_y=1, pad=False, out_activation_min=-127, out_activation_max=127, generate_bias=False, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=3, x_in=4, y_in=4, w_x=2, w_y=2, stride_x=2, stride_y=2, pad=False, out_activation_min=-127, out_activation_max=127, generate_bias=False, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=2, out_ch=8, x_in=16, y_in=16, w_x=8, w_y=8, stride_x=2, stride_y=2, pad=False, out_activation_min=-127, out_activation_max=127, generate_bias=False, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic_2' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=3, out_ch=9, x_in=9, y_in=9, w_x=6, w_y=5, stride_x=2, stride_y=1, pad=True, out_activation_min=-127, out_activation_max=127, generate_bias=True, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic_3' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=8, x_in=9, y_in=9, w_x=5, w_y=5, stride_x=1, stride_y=1, pad=False, out_activation_min=-127, out_activation_max=125, dilation_x=2, dilation_y=2, generate_bias=True, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic_4' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=1, out_ch=3, x_in=12, y_in=10, w_x=5, w_y=5, stride_x=1, stride_y=2, pad=True, out_activation_min=-127, out_activation_max=127, generate_bias=True, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic_5' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=20, out_ch=20, x_in=21, y_in=21, w_x=5, w_y=5, stride_x=1, stride_y=2, pad=False, out_activation_min=-127, out_activation_max=125, dilation_x=2, dilation_y=2, generate_bias=True, interpreter=interpreter, int4_weights=True) dataset = 'depthwise_int4_generic_6' testdata_sets[dataset] = ConvSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=4, out_ch=12, x_in=21, y_in=21, w_x=3, w_y=3, stride_x=1, stride_y=1, pad=False, out_activation_min=-127, out_activation_max=125, dilation_x=1, dilation_y=2, generate_bias=True, interpreter=interpreter, int4_weights=True) type_of_test = 'fully_connected' dataset = 'fully_connected' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=10, out_ch=6, x_in=2, y_in=1, batches=3, interpreter=interpreter) dataset = 'fully_connected_w_zp' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=10, out_ch=6, x_in=2, y_in=1, batches=3, input_scale=0.034, w_scale=0.054, bias_scale=0.00000001, output_scale=0.356, input_zp=2, output_zp=35, w_zp=15, generate_bias=False, interpreter=interpreter) dataset = 'fully_connected_mve_0' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=16, out_ch=9, x_in=1, y_in=1, batches=1, interpreter=interpreter) dataset = 'fully_connected_mve_1' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=20, out_ch=4, x_in=1, y_in=1, batches=1, interpreter=interpreter) dataset = 'fully_connected_null_bias_0' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=33, out_ch=5, batches=2, generate_bias=False, interpreter=interpreter) dataset = 'fully_connected_out_activation' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=10, out_ch=4, out_activation_min=-70, out_activation_max=100, interpreter=interpreter) dataset = 'fully_connected_int16' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=7, out_ch=11, x_in=3, y_in=3, batches=2, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-9999, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'fully_connected_int4' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=82, out_ch=21, x_in=1, y_in=1, batches=1, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, int16xint8=False, input_zp=3, output_zp=-3, interpreter=interpreter) dataset = 'fully_connected_int4_2' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=84, out_ch=23, x_in=1, y_in=1, batches=1, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, generate_bias=True, int16xint8=False, input_zp=-4, output_zp=16, interpreter=interpreter) dataset = 'fully_connected_int4_3' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=86, out_ch=25, x_in=1, y_in=1, batches=1, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, out_activation_min=-64, out_activation_max=64, int16xint8=False, input_zp=1, output_zp=0, interpreter=interpreter) dataset = 'fully_connected_int4_4' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=85, out_ch=19, x_in=1, y_in=1, batches=1, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, generate_bias=False, int16xint8=False, input_zp=-2, output_zp=0, interpreter=interpreter) dataset = 'fully_connected_int4_5' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=16, out_ch=20, bias_min=TestSettings.INT8_MIN, bias_max=TestSettings.INT8_MAX, input_zp=120, output_zp=100, interpreter=interpreter) dataset = 'fully_connected_int4_6' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, int4_weights=True, in_ch=89, out_ch=22, x_in=1, bias_min=TestSettings.INT32_MIN, bias_max=TestSettings.INT32_MAX, y_in=1, batches=1, generate_bias=False, int16xint8=False, input_zp=-127, output_zp=128, interpreter=interpreter) dataset = 'fully_connected_int16_big' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=7, out_ch=11, x_in=10, y_in=10, batches=3, out_activation_min=-1444, out_activation_max=32767, int16xint8=True, interpreter=interpreter) dataset = 'fc_int16_slow' testdata_sets[dataset] = FullyConnectedSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, in_ch=7, out_ch=11, x_in=10, y_in=8, batches=3, randmin=(TestSettings.INT16_MAX - 100), randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) type_of_test = 'avgpool' dataset = 'avgpooling' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=20, x_in=22, y_in=12, stride_x=9, stride_y=5, w_x=6, w_y=5, pad=True, interpreter=interpreter) dataset = 'avgpooling_1' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=3, x_in=9, y_in=5, stride_x=1, stride_y=2, w_x=9, w_y=5, pad=False, interpreter=interpreter) dataset = 'avgpooling_2' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=5, x_in=12, y_in=1, stride_x=1, stride_y=2, w_x=3, w_y=1, pad=True, interpreter=interpreter) dataset = 'avgpooling_3' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=9, y_in=1, stride_x=2, stride_y=1, w_x=1, w_y=1, batches=2, pad=False, interpreter=interpreter) dataset = 'avgpooling_4' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=1, y_in=20, stride_x=1, stride_y=3, w_x=1, w_y=3, batches=3, pad=True, interpreter=interpreter) dataset = 'avgpooling_5' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=1, x_in=3, y_in=3, stride_x=1, stride_y=1, w_x=1, w_y=3, pad=True, relu6=True, interpreter=interpreter) dataset = 'avgpooling_int16' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=17, x_in=6, y_in=4, stride_x=2, stride_y=1, w_x=2, w_y=3, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'avgpooling_int16_1' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=9, y_in=1, stride_x=2, stride_y=1, w_x=1, w_y=1, batches=3, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'avgpooling_int16_2' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=20, x_in=9, y_in=1, stride_x=2, stride_y=1, w_x=1, w_y=1, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'avgpooling_int16_3' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=21, x_in=1, y_in=20, stride_x=1, stride_y=3, w_x=1, w_y=3, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) type_of_test = 'maxpool' dataset = 'maxpooling' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=8, x_in=22, y_in=12, stride_x=9, stride_y=5, w_x=6, w_y=5, batches=2, pad=True, interpreter=interpreter) dataset = 'maxpooling_1' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=3, x_in=9, y_in=5, stride_x=1, stride_y=2, w_x=9, w_y=5, pad=False, interpreter=interpreter) dataset = 'maxpooling_2' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=5, x_in=12, y_in=1, stride_x=1, stride_y=2, w_x=3, w_y=1, pad=True, interpreter=interpreter) dataset = 'maxpooling_3' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=9, y_in=1, stride_x=2, stride_y=1, w_x=1, w_y=1, batches=3, pad=False, interpreter=interpreter) dataset = 'maxpooling_4' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=1, y_in=20, stride_x=1, stride_y=3, w_x=1, w_y=3, pad=True, interpreter=interpreter) dataset = 'maxpooling_5' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=20, x_in=1, y_in=1, stride_x=1, stride_y=1, w_x=1, w_y=1, pad=True, interpreter=interpreter) dataset = 'maxpooling_6' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=17, x_in=1, y_in=5, stride_x=1, stride_y=3, w_x=3, w_y=4, pad=True, interpreter=interpreter) dataset = 'maxpooling_7' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=1, x_in=4, y_in=2, stride_x=2, stride_y=2, w_x=2, w_y=2, pad=False, relu6=True, interpreter=interpreter) dataset = 'maxpool_int16' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=4, y_in=3, stride_x=2, stride_y=2, w_x=2, w_y=2, batches=3, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'maxpool_int16_1' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=2, x_in=4, y_in=5, stride_x=2, stride_y=1, w_x=3, w_y=3, batches=2, pad=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-30000, out_activation_max=30000, int16xint8=True, interpreter=interpreter) dataset = 'maxpool_int16_2' testdata_sets[dataset] = PoolingSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=3, x_in=7, y_in=7, stride_x=1, stride_y=1, w_x=3, w_y=3, pad=False, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-30000, out_activation_max=30000, int16xint8=True, interpreter=interpreter) type_of_test = 'softmax' dataset = 'softmax' testdata_sets[dataset] = SoftmaxSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, x_in=5, y_in=2, interpreter=interpreter) dataset = 'softmax_s16' testdata_sets[dataset] = SoftmaxSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, x_in=10, y_in=3, int16xint8=True, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, interpreter=interpreter) dataset = 'softmax_s8_s16' testdata_sets[dataset] = SoftmaxSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, x_in=12, y_in=2, inInt8outInt16=True, interpreter=interpreter) type_of_test = 'svdf' dataset = 'svdf' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=2, number_inputs=2, rank=8, memory_size=8, input_size=3, number_units=3, interpreter=interpreter) dataset = 'svdf_1' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=3, number_inputs=2, rank=1, memory_size=2, input_size=7, number_units=5, interpreter=interpreter) dataset = 'svdf_2' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=3, number_inputs=2, rank=2, memory_size=2, input_size=7, number_units=5, generate_bias=False, interpreter=interpreter) dataset = 'svdf_3' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=1, number_inputs=2, rank=1, memory_size=2, input_size=20, number_units=12, generate_bias=False, interpreter=interpreter) dataset = 'svdf_int8' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=1, number_inputs=2, rank=1, memory_size=2, input_size=20, number_units=12, generate_bias=False, int8_time_weights=True, interpreter=interpreter) dataset = 'svdf_int8_2' testdata_sets[dataset] = SVDFSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=2, number_inputs=3, rank=2, memory_size=3, input_size=40, number_units=13, input_zp=-12, int8_time_weights=True, interpreter=interpreter) type_of_test = 'add' dataset = 'add' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=8, x_in=4, y_in=4, randmin=TestSettings.INT8_MIN, randmax=TestSettings.INT8_MAX, interpreter=interpreter) dataset = 'add_s16' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=8, x_in=4, y_in=4, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'add_s16_spill' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=7, x_in=5, y_in=3, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=-2000, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) type_of_test = 'mul' dataset = 'mul' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=8, x_in=4, y_in=5, randmin=TestSettings.INT8_MIN, randmax=TestSettings.INT8_MAX, interpreter=interpreter) dataset = 'mul_s16' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=8, x_in=5, y_in=4, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MIN, out_activation_max=TestSettings.INT16_MAX, int16xint8=True, interpreter=interpreter) dataset = 'mul_s16_spill' testdata_sets[dataset] = AddMulSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, channels=7, x_in=5, y_in=7, randmin=TestSettings.INT16_MIN, randmax=TestSettings.INT16_MAX, out_activation_min=TestSettings.INT16_MIN, out_activation_max=1000, int16xint8=True, interpreter=interpreter) type_of_test = 'lstm' dataset = 'lstm_1' testdata_sets[dataset] = LSTMSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=1, time_steps=10, number_inputs=22, number_units=11, time_major=True, interpreter=interpreter) dataset = 'lstm_2' testdata_sets[dataset] = LSTMSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=1, time_steps=9, number_inputs=6, number_units=7, time_major=False, interpreter=interpreter) dataset = 'lstm_one_time_step' testdata_sets[dataset] = LSTMSettings(dataset, type_of_test, regenerate_weights, regenerate_input, regenerate_biases, schema_file, batches=3, time_steps=1, number_inputs=22, number_units=3, time_major=False, interpreter=interpreter) return testdata_sets def main(): if version.parse(tf.__version__) < TestSettings.REQUIRED_MINIMUM_TENSORFLOW_VERSION: print("Unsupported tensorflow version, ", version.parse(tf.__version__)) return 1 args = parse_args() testdataset = args.dataset test_type = args.testtype schema_file = args.schema_file regenerate_input = args.regenerate_input regenerate_weights = args.regenerate_weights regenerate_biases = args.regenerate_biases if args.regenerate_all: regenerate_biases = True regenerate_weights = True regenerate_input = True testdata_sets = load_testdata_sets(regenerate_input, regenerate_weights, regenerate_biases, schema_file, args.interpreter) if args.run_all_testsets: for testset_name, testset_generator in testdata_sets.items(): if test_type and testset_generator.test_type != test_type: continue print("Generating testset {}..".format(testset_name)) testset_generator.generate_data() print() # Check that all testsets have been loaded. found_test_data_sets = [] directory = 'TestCases/TestData' for dir in next(os.walk(directory))[1]: found_test_data_sets.append(dir) for testset_name in found_test_data_sets: if testset_name not in testdata_sets: print("WARNING: Testset {} in {} was not loaded".format(testset_name, directory)) elif testdataset: try: generator = testdata_sets[testdataset] except KeyError: print("WARNING: testset {} not in testset list".format(testdataset)) if test_type == 'conv' or test_type == 'depthwise_conv': generator = ConvSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'fully_connected': generator = FullyConnectedSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'avgpool' or test_type == 'maxpool': generator = PoolingSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'softmax': generator = SoftmaxSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'svdf': generator = SVDFSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'add' or test_type == 'mul': generator = AddMulSettings(testdataset, test_type, True, True, True, schema_file) elif test_type == 'lstm': generator = LSTMSettings(testdataset, test_type, True, True, True, schema_file) else: raise RuntimeError("Please specify type of test with -t") generator.generate_data() else: raise RuntimeError("Please select testdataset or use --run-all-testsets") return 0 if __name__ == '__main__': sys.exit(main())