1 /*
2 * SPDX-FileCopyrightText: Copyright 2022 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_depthwise_conv_s16.c
22 * Description: s16 version of depthwise convolution.
23 *
24 * $Date: 26 October 2022
25 * $Revision: V.2.0.1
26 *
27 * Target Processor: Cortex-M CPUs
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
34 /**
35 * @ingroup Public
36 */
37
38 /**
39 * @addtogroup NNConv
40 * @{
41 */
42
depthwise_conv_s16_mult_4_s16(const int16_t * input,const int32_t input_x,const int32_t input_y,const int32_t input_ch,const int8_t * kernel,const int32_t output_ch,const int32_t ch_mult,const int32_t kernel_x,const int32_t kernel_y,const int32_t pad_x,const int32_t pad_y,const int32_t stride_x,const int32_t stride_y,const int64_t * bias,int16_t * output,const int32_t * output_shift,const int32_t * output_mult,const int32_t output_x,const int32_t output_y,const int32_t output_activation_min,const int32_t output_activation_max)43 static void __attribute__((unused)) depthwise_conv_s16_mult_4_s16(const int16_t *input,
44 const int32_t input_x,
45 const int32_t input_y,
46 const int32_t input_ch,
47 const int8_t *kernel,
48 const int32_t output_ch,
49 const int32_t ch_mult,
50 const int32_t kernel_x,
51 const int32_t kernel_y,
52 const int32_t pad_x,
53 const int32_t pad_y,
54 const int32_t stride_x,
55 const int32_t stride_y,
56 const int64_t *bias,
57 int16_t *output,
58 const int32_t *output_shift,
59 const int32_t *output_mult,
60 const int32_t output_x,
61 const int32_t output_y,
62 const int32_t output_activation_min,
63 const int32_t output_activation_max)
64 {
65 for (int32_t in_h = -pad_y, out_h = 0, out_idx = 0; out_h < output_y; in_h += stride_y, ++out_h)
66 {
67 for (int32_t in_w = -pad_x, out_w = 0, ker_h_start = MAX(0, -in_h); out_w < output_x; in_w += stride_x, ++out_w)
68 {
69 for (int32_t in_ch = 0, out_ch = 0, ker_w_start = MAX(0, -in_w); out_ch < output_ch;
70 ++in_ch, out_ch += ch_mult)
71 {
72 for (int mult_tile = 0; mult_tile < ch_mult; mult_tile += 4)
73 {
74 int32_t out_buff32[4] = {REDUCE_MULTIPLIER(output_mult[out_ch + 0 + mult_tile]),
75 REDUCE_MULTIPLIER(output_mult[out_ch + 1 + mult_tile]),
76 REDUCE_MULTIPLIER(output_mult[out_ch + 2 + mult_tile]),
77 REDUCE_MULTIPLIER(output_mult[out_ch + 3 + mult_tile])};
78
79 int64_t out_buff[4] = {0, 0, 0, 0};
80
81 if (bias)
82 {
83 out_buff[0] = bias[out_ch + 0 + mult_tile];
84 out_buff[1] = bias[out_ch + 1 + mult_tile];
85 out_buff[2] = bias[out_ch + 2 + mult_tile];
86 out_buff[3] = bias[out_ch + 3 + mult_tile];
87 }
88
89 for (int32_t ker_h = ker_h_start; ker_h < MIN(kernel_y, input_y - in_h); ++ker_h)
90 {
91 int32_t ker_idx = ker_h * (output_ch * kernel_x) + ker_w_start * output_ch + out_ch;
92 int32_t in_idx = (in_h + ker_h) * (input_ch * input_x) + in_w * input_ch + in_ch;
93 #if defined(__ARMCC_VERSION) && (__ARMCC_VERSION >= 6010050)
94 #pragma clang loop unroll(disable)
95 #endif
96 for (int32_t ker_w = ker_w_start; ker_w < MIN(kernel_x, input_x - in_w);
97 ++ker_w, ker_idx += output_ch)
98 {
99 // TODO: Unroll of 4 with 64 bit accumulator will probably result in too much register
100 // spills. Try with unroll of 2 when enabling this.
101 int32_t in_val = input[in_idx + ker_w * input_ch];
102 out_buff[0] += in_val * kernel[ker_idx + 0 + mult_tile];
103 out_buff[1] += in_val * kernel[ker_idx + 1 + mult_tile];
104 out_buff[2] += in_val * kernel[ker_idx + 2 + mult_tile];
105 out_buff[3] += in_val * kernel[ker_idx + 3 + mult_tile];
106 }
107 }
108
109 out_buff32[0] =
110 arm_nn_requantize_s64(out_buff[0], out_buff32[0], output_shift[out_ch + 0 + mult_tile]);
111 out_buff32[1] =
112 arm_nn_requantize_s64(out_buff[1], out_buff32[1], output_shift[out_ch + 1 + mult_tile]);
113 out_buff32[2] =
114 arm_nn_requantize_s64(out_buff[2], out_buff32[2], output_shift[out_ch + 2 + mult_tile]);
115 out_buff32[3] =
116 arm_nn_requantize_s64(out_buff[3], out_buff32[3], output_shift[out_ch + 3 + mult_tile]);
117
118 out_buff32[0] = MIN(MAX(out_buff32[0], output_activation_min), output_activation_max);
119 out_buff32[1] = MIN(MAX(out_buff32[1], output_activation_min), output_activation_max);
120 out_buff32[2] = MIN(MAX(out_buff32[2], output_activation_min), output_activation_max);
121 out_buff32[3] = MIN(MAX(out_buff32[3], output_activation_min), output_activation_max);
122
123 output[out_idx++] = (int16_t)out_buff32[0];
124 output[out_idx++] = (int16_t)out_buff32[1];
125 output[out_idx++] = (int16_t)out_buff32[2];
126 output[out_idx++] = (int16_t)out_buff32[3];
127 }
128 }
129 }
130 }
131 }
132
depthwise_conv_s16_generic_s16(const int16_t * input,const uint16_t input_batches,const uint16_t input_x,const uint16_t input_y,const uint16_t input_ch,const int8_t * kernel,const uint16_t ch_mult,const uint16_t kernel_x,const uint16_t kernel_y,const uint16_t pad_x,const uint16_t pad_y,const uint16_t stride_x,const uint16_t stride_y,const int64_t * bias,int16_t * output,const int32_t * output_shift,const int32_t * output_mult,const uint16_t output_x,const uint16_t output_y,const int32_t output_activation_min,const int32_t output_activation_max,const uint16_t dilation_x,const uint16_t dilation_y)133 static void depthwise_conv_s16_generic_s16(const int16_t *input,
134 const uint16_t input_batches,
135 const uint16_t input_x,
136 const uint16_t input_y,
137 const uint16_t input_ch,
138 const int8_t *kernel,
139 const uint16_t ch_mult,
140 const uint16_t kernel_x,
141 const uint16_t kernel_y,
142 const uint16_t pad_x,
143 const uint16_t pad_y,
144 const uint16_t stride_x,
145 const uint16_t stride_y,
146 const int64_t *bias,
147 int16_t *output,
148 const int32_t *output_shift,
149 const int32_t *output_mult,
150 const uint16_t output_x,
151 const uint16_t output_y,
152 const int32_t output_activation_min,
153 const int32_t output_activation_max,
154 const uint16_t dilation_x,
155 const uint16_t dilation_y)
156
157 {
158 for (int i_batch = 0; i_batch < input_batches; i_batch++)
159 {
160 for (int i_out_y = 0; i_out_y < output_y; i_out_y++)
161 {
162 const int16_t base_idx_y = (i_out_y * stride_y) - pad_y;
163 for (int i_out_x = 0; i_out_x < output_x; i_out_x++)
164 {
165 const int16_t base_idx_x = (i_out_x * stride_x) - pad_x;
166 for (int i_input_ch = 0; i_input_ch < input_ch; i_input_ch++)
167 {
168 for (int i_ch_mult = 0; i_ch_mult < ch_mult; i_ch_mult++)
169 {
170 const int idx_out_ch = i_ch_mult + i_input_ch * ch_mult;
171
172 const int32_t reduced_multiplier = REDUCE_MULTIPLIER(output_mult[idx_out_ch]);
173 int64_t acc_0 = 0;
174
175 int ker_y_start;
176 int ker_x_start;
177 int ker_y_end;
178 int ker_x_end;
179
180 if (dilation_x > 1)
181 {
182 const int32_t start_x_max = (-base_idx_x + dilation_x - 1) / dilation_x;
183 ker_x_start = MAX(0, start_x_max);
184 const int32_t end_min_x = (input_x - base_idx_x + dilation_x - 1) / dilation_x;
185 ker_x_end = MIN(kernel_x, end_min_x);
186 }
187 else
188 {
189 ker_x_start = MAX(0, -base_idx_x);
190 ker_x_end = MIN(kernel_x, input_x - base_idx_x);
191 }
192
193 if (dilation_y > 1)
194 {
195 const int32_t start_y_max = (-base_idx_y + dilation_y - 1) / dilation_y;
196 ker_y_start = MAX(0, start_y_max);
197 const int32_t end_min_y = (input_y - base_idx_y + dilation_y - 1) / dilation_y;
198 ker_y_end = MIN(kernel_y, end_min_y);
199 }
200 else
201 {
202 ker_y_start = MAX(0, -base_idx_y);
203 ker_y_end = MIN(kernel_y, input_y - base_idx_y);
204 }
205
206 if (bias)
207 {
208 acc_0 = bias[idx_out_ch];
209 }
210
211 for (int i_ker_y = ker_y_start; i_ker_y < ker_y_end; i_ker_y++)
212 {
213 const int32_t idx_y = base_idx_y + dilation_y * i_ker_y;
214 for (int i_ker_x = ker_x_start; i_ker_x < ker_x_end; i_ker_x++)
215 {
216 const int32_t idx_x = base_idx_x + dilation_x * i_ker_x;
217 int32_t idx_0 = (idx_y * input_x + idx_x) * input_ch + i_input_ch;
218 int32_t ker_idx_0 = (i_ker_y * kernel_x + i_ker_x) * (input_ch * ch_mult) + idx_out_ch;
219
220 acc_0 += input[idx_0] * kernel[ker_idx_0];
221 }
222 }
223
224 /* Requantize and clamp output to provided range */
225 int32_t result = arm_nn_requantize_s64(acc_0, reduced_multiplier, output_shift[idx_out_ch]);
226 result = MAX(result, output_activation_min);
227 result = MIN(result, output_activation_max);
228 *output++ = (int16_t)result;
229 }
230 }
231 }
232 }
233 /* Advance to the next batch */
234 input += (input_x * input_y * input_ch);
235 }
236 }
237
238 /*
239 * Basic s16 depthwise convolution function.
240 *
241 * Refer header file for details.
242 *
243 */
arm_depthwise_conv_s16(const cmsis_nn_context * ctx,const cmsis_nn_dw_conv_params * dw_conv_params,const cmsis_nn_per_channel_quant_params * quant_params,const cmsis_nn_dims * input_dims,const int16_t * input,const cmsis_nn_dims * filter_dims,const int8_t * kernel,const cmsis_nn_dims * bias_dims,const int64_t * bias,const cmsis_nn_dims * output_dims,int16_t * output)244 arm_cmsis_nn_status arm_depthwise_conv_s16(const cmsis_nn_context *ctx,
245 const cmsis_nn_dw_conv_params *dw_conv_params,
246 const cmsis_nn_per_channel_quant_params *quant_params,
247 const cmsis_nn_dims *input_dims,
248 const int16_t *input,
249 const cmsis_nn_dims *filter_dims,
250 const int8_t *kernel,
251 const cmsis_nn_dims *bias_dims,
252 const int64_t *bias,
253 const cmsis_nn_dims *output_dims,
254 int16_t *output)
255 {
256 const uint16_t dilation_x = dw_conv_params->dilation.w;
257 const uint16_t dilation_y = dw_conv_params->dilation.h;
258
259 (void)bias_dims;
260 (void)ctx;
261
262 depthwise_conv_s16_generic_s16(input,
263 input_dims->n,
264 input_dims->w,
265 input_dims->h,
266 input_dims->c,
267 kernel,
268 dw_conv_params->ch_mult,
269 filter_dims->w,
270 filter_dims->h,
271 dw_conv_params->padding.w,
272 dw_conv_params->padding.h,
273 dw_conv_params->stride.w,
274 dw_conv_params->stride.h,
275 bias,
276 output,
277 quant_params->shift,
278 quant_params->multiplier,
279 output_dims->w,
280 output_dims->h,
281 dw_conv_params->activation.min,
282 dw_conv_params->activation.max,
283 dilation_x,
284 dilation_y);
285
286 /* Return to application */
287 return ARM_CMSIS_NN_SUCCESS;
288 }
289
290 /**
291 * @} end of NNConv group
292 */
293
