1# SPDX-FileCopyrightText: Copyright 2010-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
2#
3# SPDX-License-Identifier: Apache-2.0
4#
5# Licensed under the Apache License, Version 2.0 (the License); you may
6# not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an AS IS BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17from test_settings import TestSettings
18
19import tensorflow as tf
20import numpy as np
21import math
22import tf_keras as keras
23
24
25class ConvSettings(TestSettings):
26
27    def __init__(self,
28                 dataset,
29                 testtype,
30                 regenerate_weights,
31                 regenerate_input,
32                 regenerate_biases,
33                 schema_file,
34                 in_ch=1,
35                 out_ch=1,
36                 x_in=7,
37                 y_in=7,
38                 w_x=3,
39                 w_y=3,
40                 stride_x=2,
41                 stride_y=2,
42                 groups=1,
43                 pad=True,
44                 randmin=TestSettings.INT8_MIN,
45                 randmax=TestSettings.INT8_MAX,
46                 batches=1,
47                 generate_bias=True,
48                 relu6=False,
49                 out_activation_min=None,
50                 out_activation_max=None,
51                 int16xint8=False,
52                 int16xint8_int32=False,
53                 bias_min=TestSettings.INT32_MIN,
54                 bias_max=TestSettings.INT32_MAX,
55                 dilation_x=1,
56                 dilation_y=1,
57                 interpreter="tensorflow",
58                 int4_weights=False,
59                 weights_min=TestSettings.INT32_MIN,
60                 weights_max=TestSettings.INT32_MAX):
61        super().__init__(dataset,
62                         testtype,
63                         regenerate_weights,
64                         regenerate_input,
65                         regenerate_biases,
66                         schema_file,
67                         in_ch,
68                         out_ch,
69                         x_in,
70                         y_in,
71                         w_x,
72                         w_y,
73                         stride_x,
74                         stride_y,
75                         pad,
76                         randmin,
77                         randmax,
78                         batches,
79                         generate_bias=generate_bias,
80                         relu6=relu6,
81                         out_activation_min=out_activation_min,
82                         out_activation_max=out_activation_max,
83                         int16xint8=int16xint8,
84                         bias_min=bias_min,
85                         bias_max=bias_max,
86                         dilation_x=dilation_x,
87                         dilation_y=dilation_y,
88                         interpreter=interpreter,
89                         int4_weights=int4_weights)
90
91        self.scaling_factors = []
92        self.groups = groups
93
94        self.weights_min = weights_min
95        self.weights_max = weights_max
96
97        if int16xint8_int32:
98            if not self.is_int16xint8:
99                raise RuntimeError("ERROR: int16x8 with int32 bias only relevant for int16x8")
100            if not self.test_type == 'conv':
101                raise RuntimeError("ERROR: int16x8 with int32 bias only supported for conv")
102        self.int16xint8_int32 = int16xint8_int32
103
104        if self.test_type == 'depthwise_conv':
105            self.channel_multiplier = self.output_ch // self.input_ch
106            if self.output_ch % self.input_ch != 0:
107                raise RuntimeError("out channel ({}) is not multiple of in channel ({})".format(out_ch, in_ch))
108            if groups != 1:
109                raise RuntimeError("ERROR: Groups cannot be used for depthwise convolution")
110        else:
111            self.channel_multiplier = 0
112
113        self.filter_ch = in_ch // groups
114        if in_ch % groups != 0:
115            raise RuntimeError("ERROR: Input channels {} must be an even multiple of groups {}".format(in_ch, groups))
116        if out_ch % groups != 0:
117            raise RuntimeError("ERROR: Output channels {} must be an even multiple of groups {}".format(out_ch, groups))
118
119        if self.int4_weights:
120            if self.test_type == 'conv':
121                self.json_template = "TestCases/Common/conv2d_s4_weights_template.json"
122            elif self.test_type == 'depthwise_conv':
123                self.json_template = "TestCases/Common/dw_s4_weights_template.json"
124
125    def write_c_config_header(self) -> None:
126        super().write_c_config_header()
127
128        filename = self.config_data
129        filepath = self.headers_dir + filename
130        prefix = self.testdataset.upper()
131
132        with open(filepath, "a") as f:
133            self.write_common_config(f, prefix)
134            if self.test_type == 'depthwise_conv':
135                f.write("#define {}_CH_MULT {}\n".format(prefix, self.channel_multiplier))
136            f.write("#define {}_INPUT_OFFSET {}\n".format(prefix, -self.input_zero_point))
137            f.write("#define {}_OUTPUT_OFFSET {}\n".format(prefix, self.output_zero_point))
138            f.write("#define {}_DILATION_X {}\n".format(prefix, self.dilation_x))
139            f.write("#define {}_DILATION_Y {}\n".format(prefix, self.dilation_y))
140            if self.groups != 1:
141                f.write("#define {}_FILTER_CH {}\n".format(prefix, self.filter_ch))
142            if self.test_type == 'transpose_conv':
143                f.write("#define {}_PAD_X_WITH_OFFSET {}\n".format(prefix, self.pad_x_with_offset))
144                f.write("#define {}_PAD_Y_WITH_OFFSET {}\n".format(prefix, self.pad_y_with_offset))
145
146    def generate_quantize_per_channel_multiplier(self):
147        num_channels = self.output_ch
148        per_channel_multiplier = []
149        per_channel_shift = []
150
151        if len(self.scaling_factors) != num_channels:
152            raise RuntimeError("Missing scaling factors")
153
154        for i in range(num_channels):
155            effective_output_scale = self.input_scale * self.scaling_factors[i] / self.output_scale
156            (quantized_multiplier, shift) = self.quantize_scale(effective_output_scale)
157
158            per_channel_multiplier.append(quantized_multiplier)
159            per_channel_shift.append(shift)
160
161        return per_channel_multiplier, per_channel_shift
162
163    def generate_int4_scale(self, scale, shift, input_scale):
164        self.output_scale = scale
165        self.output_zp = shift
166        self.input_scale = input_scale
167        self.scaling_factors = np.random.uniform(0.001, 0.01, [self.output_ch]).tolist()
168        per_channel_multiplier, per_channel_shift = self.generate_quantize_per_channel_multiplier()
169
170        while any((x > 31 or x < -31) for x in per_channel_shift):
171            self.output_scale = self.output_scale / 10
172            per_channel_multiplier, per_channel_shift = self.generate_quantize_per_channel_multiplier()
173
174        return self.output_scale, self.output_zp
175
176    # TODO
177    def quantize_float_data(self, data=None, quantization_bit_range=8, quantization_type="affine", tf_tensor=False):
178        if data is not None:
179            if tf_tensor:
180                data = data.numpy()
181            data_max = np.amax(data)
182            data_min = np.amin(data)
183
184            if quantization_type.lower() == "affine":
185                data_min = min(data_min, 0.0)
186                data_max = max(data_max, 0.0)
187
188                scale = (data_max - data_min) / (pow(2, quantization_bit_range) - 1)
189                zero_point = -(round(data_max * scale)) - pow(2, quantization_bit_range - 1)
190                zero_point = max(zero_point, pow(quantization_bit_range - 1) - 1)
191                zero_point = min(zero_point, -pow(quantization_bit_range - 1))
192
193            elif quantization_type.lower() == "symmetric":
194                absolute_max = max(abs(data_min), abs(data_max))
195                scale = absolute_max / (pow(2, quantization_bit_range - 1) - 1)
196                zero_point = 0
197
198            else:
199                raise RuntimeError("Quantization scheme not supported")
200
201            scale = 0.1 if scale == 0 else scale
202            quantized_data = [(x // scale) + zero_point for x in data]
203            return tf.convert_to_tensor(quantized_data), scale, zero_point
204
205    def generate_data(self, input_data=None, weights=None, biases=None) -> None:
206        if self.is_int16xint8:
207            inttype = tf.int16
208            datatype = "int16_t"
209            bias_datatype = "int32_t" if self.int16xint8_int32 else "int64_t"
210        else:
211            inttype = tf.int8
212            datatype = "int8_t"
213            bias_datatype = "int32_t"
214
215        input_data = self.get_randomized_input_data(input_data)
216        biases = self.get_randomized_bias_data(biases)
217
218        if self.test_type == 'conv' or self.test_type == 'transpose_conv':
219            out_channel = self.output_ch
220        elif self.test_type == 'depthwise_conv':
221            out_channel = self.channel_multiplier
222
223        if self.int4_weights:
224            w_shape = [self.filter_y * self.filter_x * self.input_ch * out_channel]
225
226            if weights is not None:
227                weights = tf.reshape(weights, w_shape)
228            else:
229                weights = self.get_randomized_data(w_shape,
230                                                   self.kernel_table_file,
231                                                   minrange=TestSettings.INT4_MIN,
232                                                   maxrange=TestSettings.INT4_MAX,
233                                                   decimals=1,
234                                                   regenerate=self.regenerate_new_weights)
235
236            input_scale = 0.046774
237            input_zp = -128
238
239            if w_shape[0] % 2:
240                weights = np.append(weights, [0])
241
242            if self.test_type == 'depthwise_conv':
243                bias_scale = [64751.269531] * self.output_ch
244                bias_zp = [0] * self.output_ch
245                if self.generate_bias:
246                    output_scale, output_zp = self.generate_int4_scale(4684910.0, -2, input_scale)
247                else:
248                    output_scale = 0.525255
249                    output_zp = 2
250            else:
251                quant_bias, bias_scale, bias_zp = self.quantize_float_data(
252                    biases, quantization_bit_range=8, quantization_type="symmetric", tf_tensor=not self.generate_bias)
253                bias_scale = [bias_scale] * self.output_ch
254                bias_zp = [bias_zp] * self.output_ch
255
256                output_scale = np.random.uniform(0.02, 0.06)
257                output_zp = 0
258
259            scaling_factors = np.random.uniform(0.001, 0.01, [self.output_ch]).tolist()
260            w_zp = [0] * self.output_ch
261
262            if self.has_padding:
263                # TODO dilation with padding
264                output_x = math.ceil(float(self.x_input) / float(self.stride_x))
265                output_y = math.ceil(float(self.y_input) / float(self.stride_y))
266            else:
267                dilation_filter_x = (self.filter_x - 1) * (self.dilation_x - 1)
268                dilation_filter_y = (self.filter_y - 1) * (self.dilation_y - 1)
269
270                output_x = math.ceil(float(self.x_input - self.filter_x - dilation_filter_x + 1) / float(self.stride_x))
271                output_y = math.ceil(float(self.y_input - self.filter_y - dilation_filter_y + 1) / float(self.stride_y))
272
273            self.json_replacements = {
274                "batches": self.batches,
275                "input_ch": self.input_ch,
276                "output_ch": self.output_ch,
277                "input_x": self.x_input,
278                "input_y": self.y_input,
279                "weight_x": self.filter_x,
280                "weight_y": self.filter_y,
281                "output_x": output_x,
282                "output_y": output_y,
283                "input_scale": input_scale,
284                "input_zp": input_zp,
285                "w_scale": scaling_factors,
286                "w_zp": w_zp,
287                "bias_scale": bias_scale,
288                "bias_zp": bias_zp,
289                "output_scale": output_scale,
290                "output_zp": output_zp,
291                "stride_x": self.stride_x,
292                "stride_y": self.stride_y,
293                "dilation_x": self.dilation_x,
294                "dilation_y": self.dilation_y,
295                "type_pad": self.padding,
296                "ch_mult": self.channel_multiplier
297            }
298
299            # Pack weights
300            temp = np.reshape(weights, (len(weights) // 2, 2)).astype(np.uint8)
301            temp = 0xff & ((0xf0 & (temp[:, 1] << 4)) | (temp[:, 0] & 0xf))
302            weights = tf.convert_to_tensor(temp)
303
304            # Generate tflite model
305            if self.test_type == 'depthwise_conv':
306                generated_json = self.generate_json_from_template(
307                    None, weights, int8_time_weights=True, bias_data=biases, bias_buffer=3)
308            else:
309                generated_json = self.generate_json_from_template(weights, int8_time_weights=False,
310                                                                  bias_data=quant_bias, bias_buffer=2)
311
312            self.flatc_generate_tflite(generated_json, self.schema_file)
313
314            filter_index = 1
315            bias_index = 2
316
317        else:
318            if self.test_type == 'transpose_conv':
319                weight_shape = [self.filter_y, self.filter_x, out_channel, self.input_ch]
320            else:
321                weight_shape = [self.filter_y, self.filter_x, self.filter_ch, out_channel]
322
323            if weights is not None:
324                weights = tf.reshape(weights, weight_shape)
325            else:
326                weights = self.get_randomized_data(weight_shape,
327                                                   self.kernel_table_file,
328                                                   minrange=self.weights_min,
329                                                   maxrange=self.weights_max,
330                                                   decimals=1,
331                                                   regenerate=self.regenerate_new_weights)
332
333            # Create a one layer Keras model.
334            model = keras.models.Sequential()
335            input_shape = (self.batches, self.y_input, self.x_input, self.input_ch)
336            model.add(keras.layers.InputLayer(input_shape=input_shape[1:], batch_size=self.batches))
337            if self.test_type == 'conv':
338                conv_layer = keras.layers.Conv2D(self.output_ch,
339                                                 kernel_size=(self.filter_y, self.filter_x),
340                                                 strides=(self.stride_y, self.stride_x),
341                                                 padding=self.padding,
342                                                 input_shape=input_shape[1:],
343                                                 dilation_rate=(self.dilation_y, self.dilation_x),
344                                                 groups=self.groups,
345                                                 use_bias=self.generate_bias)
346                model.add(conv_layer)
347                if self.generate_bias:
348                    conv_layer.set_weights([weights, biases])
349                else:
350                    conv_layer.set_weights([weights])
351            elif self.test_type == 'depthwise_conv':
352                depthwise_layer = keras.layers.DepthwiseConv2D(kernel_size=(self.filter_y, self.filter_x),
353                                                               strides=(self.stride_y, self.stride_x),
354                                                               padding=self.padding,
355                                                               depth_multiplier=self.channel_multiplier,
356                                                               input_shape=input_shape[1:],
357                                                               dilation_rate=(self.dilation_y, self.dilation_x),
358                                                               use_bias=self.generate_bias)
359                model.add(depthwise_layer)
360                if self.generate_bias:
361                    depthwise_layer.set_weights([weights, biases])
362                else:
363                    depthwise_layer.set_weights([weights])
364            elif self.test_type == 'transpose_conv':
365                transposed_conv_layer = keras.layers.Conv2DTranspose(self.output_ch,
366                                                                     kernel_size=(self.filter_y, self.filter_x),
367                                                                     strides=(self.stride_y, self.stride_x),
368                                                                     padding=self.padding,
369                                                                     input_shape=input_shape[1:],
370                                                                     dilation_rate=(self.dilation_y,
371                                                                                    self.dilation_x),
372                                                                     use_bias=self.generate_bias)
373                model.add(transposed_conv_layer)
374                if self.generate_bias:
375                    transposed_conv_layer.set_weights([weights, biases])
376                else:
377                    transposed_conv_layer.set_weights([weights])
378
379            if self.test_type == 'transpose_conv' and self.generate_bias:
380                filter_index = 3
381                bias_index = 2
382            elif self.is_int16xint8 and self.generate_bias:
383                filter_index = 1
384                bias_index = 2
385            else:
386                filter_index = 2
387                bias_index = 1
388
389            self.convert_model(model, inttype, int16x8_int32bias=self.int16xint8_int32)
390
391        interpreter = self.interpret_model(input_data, inttype)
392
393        all_layers_details = interpreter.get_tensor_details()
394        filter_layer = all_layers_details[filter_index]
395
396        if not self.int4_weights and not self.generate_bias:
397            bias_layer = None
398            biases = []
399        else:
400            bias_layer = all_layers_details[bias_index]
401
402        if self.int4_weights:
403            expected_weight_size = math.ceil(interpreter.get_tensor(filter_layer['index']).size / 2)
404        else:
405            expected_weight_size = interpreter.get_tensor(filter_layer['index']).size
406
407        if weights.numpy().size != expected_weight_size or \
408                (self.generate_bias and biases.numpy().size != interpreter.get_tensor(bias_layer['index']).size):
409            raise RuntimeError(f"Dimension mismatch for {self.testdataset}")
410
411        output_details = interpreter.get_output_details()
412
413        self.x_output = output_details[0]['shape'][2]
414        self.y_output = output_details[0]['shape'][1]
415
416        if self.test_type == 'transpose_conv':
417            self.calculate_padding(self.x_input, self.y_input, self.x_output, self.y_output)
418        else:
419            self.calculate_padding(self.x_output, self.y_output, self.x_input, self.y_input)
420
421        self.generate_c_array(self.input_data_file_prefix, input_data, datatype=datatype)
422        self.generate_c_array(
423            self.weight_data_file_prefix, interpreter.get_tensor(filter_layer['index']), pack=self.int4_weights)
424
425        self.scaling_factors = filter_layer['quantization_parameters']['scales']
426        per_channel_multiplier, per_channel_shift = self.generate_quantize_per_channel_multiplier()
427        self.generate_c_array("output_mult", per_channel_multiplier, datatype='int32_t')
428        self.generate_c_array("output_shift", per_channel_shift, datatype='int32_t')
429
430        if self.generate_bias:
431            self.generate_c_array(
432                self.bias_data_file_prefix, interpreter.get_tensor(bias_layer['index']), bias_datatype)
433        else:
434            self.generate_c_array(
435                self.bias_data_file_prefix, biases, bias_datatype)
436
437        # Generate reference
438        interpreter.invoke()
439        output_data = interpreter.get_tensor(output_details[0]["index"])
440        self.generate_c_array(self.output_data_file_prefix,
441                              np.clip(output_data, self.out_activation_min, self.out_activation_max),
442                              datatype=datatype)
443
444        self.write_c_config_header()
445        self.write_c_header_wrapper()
446