1 /*
2 * SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_avgpool_s8.c
22 * Description: Pooling function implementations
23 *
24 * $Date: 27 November 2023
25 * $Revision: V.3.3.0
26 *
27 * Target : Arm(R) M-Profile Architecture
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
34 #if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI)
scale_q31_to_q7_and_clamp(const int32_t * buffer,int8_t * target,int32_t length,const int32_t count,const int act_min,const int act_max)35 static void scale_q31_to_q7_and_clamp(const int32_t *buffer,
36 int8_t *target,
37 int32_t length,
38 const int32_t count,
39 const int act_min,
40 const int act_max)
41 {
42 const int half_count = count / 2;
43
44 for (int i = 0; i < length; i++)
45 {
46 int32_t sum = buffer[i] > 0 ? (buffer[i] + half_count) : (buffer[i] - half_count);
47 sum = sum / count;
48 sum = MAX(sum, act_min);
49 sum = MIN(sum, act_max);
50
51 target[i] = (int8_t)sum;
52 }
53 }
54 #endif
55
56 /**
57 * @ingroup Public
58 */
59
60 /**
61 * @addtogroup Pooling
62 * @{
63 */
64
65 /*
66 * s8 average pooling function
67 *
68 * Refer to header file for details.
69 *
70 */
71
72 #if defined(ARM_MATH_MVEI)
73
arm_avgpool_s8(const cmsis_nn_context * ctx,const cmsis_nn_pool_params * pool_params,const cmsis_nn_dims * input_dims,const int8_t * src,const cmsis_nn_dims * filter_dims,const cmsis_nn_dims * output_dims,int8_t * dst)74 arm_cmsis_nn_status arm_avgpool_s8(const cmsis_nn_context *ctx,
75 const cmsis_nn_pool_params *pool_params,
76 const cmsis_nn_dims *input_dims,
77 const int8_t *src,
78 const cmsis_nn_dims *filter_dims,
79 const cmsis_nn_dims *output_dims,
80 int8_t *dst)
81 {
82 (void)ctx;
83 const int32_t input_y = input_dims->h;
84 const int32_t input_x = input_dims->w;
85 const int32_t output_y = output_dims->h;
86 const int32_t output_x = output_dims->w;
87 const int32_t stride_y = pool_params->stride.h;
88 const int32_t stride_x = pool_params->stride.w;
89 const int32_t kernel_y = filter_dims->h;
90 const int32_t kernel_x = filter_dims->w;
91 const int32_t pad_y = pool_params->padding.h;
92 const int32_t pad_x = pool_params->padding.w;
93 const int32_t act_min = pool_params->activation.min;
94 const int32_t act_max = pool_params->activation.max;
95 const int32_t ch_src = input_dims->c;
96 const int32_t batch_input = input_x * input_y * ch_src;
97 const int32_t batch_output = output_x * output_y * ch_src;
98 int32_t batch_cnt = input_dims->n;
99
100 if (batch_cnt < 1)
101 {
102 return ARM_CMSIS_NN_ARG_ERROR;
103 }
104
105 while (batch_cnt)
106 {
107 for (int i_y = 0; i_y < output_y; i_y++)
108 {
109 for (int i_x = 0; i_x < output_x; i_x++)
110 {
111 const int32_t k_y_start = MAX(0, i_y * stride_y - pad_y);
112 const int32_t k_y_end = MIN(i_y * stride_y - pad_y + kernel_y, input_y);
113
114 const int32_t k_x_start = MAX(0, i_x * stride_x - pad_x);
115 const int32_t k_x_end = MIN(i_x * stride_x - pad_x + kernel_x, input_x);
116
117 const int8_t *src_base = src;
118 int8_t *out = &dst[ch_src * (i_x + i_y * output_x)];
119
120 int32_t ch_count = (ch_src + 15) / 16;
121 int32_t channels = ch_src;
122
123 while (ch_count > 0)
124 {
125 int8x16_t temp;
126 int16x8_t temp_lo, temp_hi;
127 int32x4_t temp_lo_lo, temp_lo_hi, temp_hi_lo, temp_hi_hi;
128 int32_t count = 0;
129
130 int32x4_t sum_1 = vdupq_n_s32(0);
131 int32x4_t sum_2 = vdupq_n_s32(0);
132 int32x4_t sum_3 = vdupq_n_s32(0);
133 int32x4_t sum_4 = vdupq_n_s32(0);
134 // Load store tail predicate
135 const mve_pred16_t ld_st_p = vctp8q(channels);
136 channels -= 16;
137
138 for (int k_y = k_y_start; k_y < k_y_end; k_y++)
139 {
140 for (int k_x = k_x_start; k_x < k_x_end; k_x++)
141 {
142 const int8_t *src_inner = src_base + (ch_src * (k_x + k_y * input_x));
143 temp = vldrbq_z_s8(src_inner, ld_st_p);
144
145 temp_lo = vmovlbq_s8(temp);
146 temp_hi = vmovltq_s8(temp);
147
148 temp_lo_lo = vmovlbq_s16(temp_lo);
149 temp_lo_hi = vmovltq_s16(temp_lo);
150
151 temp_hi_lo = vmovlbq_s16(temp_hi);
152 temp_hi_hi = vmovltq_s16(temp_hi);
153
154 sum_1 = vaddq_s32(sum_1, temp_lo_lo);
155 sum_2 = vaddq_s32(sum_2, temp_lo_hi);
156 sum_3 = vaddq_s32(sum_3, temp_hi_lo);
157 sum_4 = vaddq_s32(sum_4, temp_hi_hi);
158
159 count++;
160 }
161 }
162
163 // Prevent static code issue DIVIDE_BY_ZERO.
164 if (count == 0)
165 {
166 return ARM_CMSIS_NN_ARG_ERROR;
167 }
168
169 // Perform the following operation
170 // sum = sum > 0 ? (sum + count / 2) / count : (sum - count / 2) / count;
171 const int32_t half_count = count / 2;
172 // Predicate for 'sum > 0' operation
173 mve_pred16_t p = vcmpgtq_n_s32(sum_1, 0);
174 sum_1 = vaddq_m_n_s32(sum_1, sum_1, half_count, p);
175 sum_1 = vsubq_m_n_s32(sum_1, sum_1, half_count, ~p);
176
177 p = vcmpgtq_n_s32(sum_2, 0);
178 sum_2 = vaddq_m_n_s32(sum_2, sum_2, half_count, p);
179 sum_2 = vsubq_m_n_s32(sum_2, sum_2, half_count, ~p);
180
181 p = vcmpgtq_n_s32(sum_3, 0);
182 sum_3 = vaddq_m_n_s32(sum_3, sum_3, half_count, p);
183 sum_3 = vsubq_m_n_s32(sum_3, sum_3, half_count, ~p);
184
185 p = vcmpgtq_n_s32(sum_4, 0);
186 sum_4 = vaddq_m_n_s32(sum_4, sum_4, half_count, p);
187 sum_4 = vsubq_m_n_s32(sum_4, sum_4, half_count, ~p);
188
189 for (int i = 0; i < 4; i++)
190 {
191 sum_1[i] = sum_1[i] / count;
192 sum_2[i] = sum_2[i] / count;
193 sum_3[i] = sum_3[i] / count;
194 sum_4[i] = sum_4[i] / count;
195 }
196
197 sum_1 = vmaxq_s32(sum_1, vdupq_n_s32(act_min));
198 sum_1 = vminq_s32(sum_1, vdupq_n_s32(act_max));
199
200 sum_2 = vmaxq_s32(sum_2, vdupq_n_s32(act_min));
201 sum_2 = vminq_s32(sum_2, vdupq_n_s32(act_max));
202
203 sum_3 = vmaxq_s32(sum_3, vdupq_n_s32(act_min));
204 sum_3 = vminq_s32(sum_3, vdupq_n_s32(act_max));
205
206 sum_4 = vmaxq_s32(sum_4, vdupq_n_s32(act_min));
207 sum_4 = vminq_s32(sum_4, vdupq_n_s32(act_max));
208
209 temp_lo = vmovnbq_s32(temp_lo, sum_1);
210 temp_lo = vmovntq_s32(temp_lo, sum_2);
211
212 temp_hi = vmovnbq_s32(temp_hi, sum_3);
213 temp_hi = vmovntq_s32(temp_hi, sum_4);
214
215 temp = vmovnbq_s16(temp, temp_lo);
216 temp = vmovntq_s16(temp, temp_hi);
217
218 vstrbq_p_s8(out, temp, ld_st_p);
219 out += 16;
220
221 ch_count--;
222 src_base += 16;
223 }
224 }
225 }
226 src += batch_input;
227 dst += batch_output;
228
229 batch_cnt--;
230 }
231
232 return ARM_CMSIS_NN_SUCCESS;
233 }
234
235 #else
arm_avgpool_s8(const cmsis_nn_context * ctx,const cmsis_nn_pool_params * pool_params,const cmsis_nn_dims * input_dims,const int8_t * src,const cmsis_nn_dims * filter_dims,const cmsis_nn_dims * output_dims,int8_t * dst)236 arm_cmsis_nn_status arm_avgpool_s8(const cmsis_nn_context *ctx,
237 const cmsis_nn_pool_params *pool_params,
238 const cmsis_nn_dims *input_dims,
239 const int8_t *src,
240 const cmsis_nn_dims *filter_dims,
241 const cmsis_nn_dims *output_dims,
242 int8_t *dst)
243 {
244 const int32_t input_y = input_dims->h;
245 const int32_t input_x = input_dims->w;
246 const int32_t output_y = output_dims->h;
247 const int32_t output_x = output_dims->w;
248 const int32_t stride_y = pool_params->stride.h;
249 const int32_t stride_x = pool_params->stride.w;
250 const int32_t kernel_y = filter_dims->h;
251 const int32_t kernel_x = filter_dims->w;
252 const int32_t pad_y = pool_params->padding.h;
253 const int32_t pad_x = pool_params->padding.w;
254 const int32_t act_min = pool_params->activation.min;
255 const int32_t act_max = pool_params->activation.max;
256 const int32_t ch_src = input_dims->c;
257 int32_t batch_cnt = input_dims->n;
258
259 if (batch_cnt < 1)
260 {
261 return ARM_CMSIS_NN_ARG_ERROR;
262 }
263
264 if (ctx->buf == NULL && arm_avgpool_s8_get_buffer_size(output_dims->w, input_dims->c))
265 {
266 return ARM_CMSIS_NN_ARG_ERROR;
267 }
268
269 #if defined(ARM_MATH_DSP)
270 /* Run the following code for CPU's with DSP extension
271 */
272 const int32_t batch_size = input_x * input_y * ch_src;
273 int32_t *buffer = (int32_t *)ctx->buf;
274
275 while (batch_cnt)
276 {
277 for (int i_y = 0, idx_y = -pad_y; i_y < output_y; idx_y += stride_y, i_y++)
278 {
279 for (int i_x = 0, idx_x = -pad_x; i_x < output_x; idx_x += stride_x, i_x++)
280 {
281 /* Condition for kernel start dimension:
282 (base_idx_<x,y> + kernel_<x,y>_start) >= 0 */
283 const int32_t kernel_y_start = MAX(0, -idx_y);
284 const int32_t kernel_x_start = MAX(0, -idx_x);
285
286 /* Condition for kernel end dimension:
287 (base_idx_<x,y> + kernel_<x,y>_end) < dim_src_<width,height> */
288 const int32_t kernel_y_end = MIN(kernel_y, input_y - idx_y);
289 const int32_t kernel_x_end = MIN(kernel_x, input_x - idx_x);
290
291 int count = 0;
292
293 for (int k_y = kernel_y_start; k_y < kernel_y_end; k_y++)
294 {
295 for (int k_x = kernel_x_start; k_x < kernel_x_end; k_x++)
296 {
297 const int8_t *start = src + ch_src * (k_x + idx_x + (k_y + idx_y) * input_x);
298
299 if (count == 0)
300 {
301 for (int i = 0; i < ch_src; i++)
302 {
303 buffer[i] = start[i];
304 }
305 }
306 else
307 {
308 for (int i = 0; i < ch_src; i++)
309 {
310 buffer[i] = QADD(start[i], buffer[i]);
311 }
312 }
313 count++;
314 }
315 }
316
317 // Prevent static code issue DIVIDE_BY_ZERO.
318 if (count == 0)
319 {
320 return ARM_CMSIS_NN_ARG_ERROR;
321 }
322
323 scale_q31_to_q7_and_clamp(buffer, dst, ch_src, count, act_min, act_max);
324 dst += ch_src;
325 }
326 }
327 src += batch_size;
328
329 batch_cnt--;
330 }
331
332 #else
333
334 /* Reference C code adapted from CMSIS-NN arm_avepool_q7_HWC.
335 */
336 const int32_t batch_input = input_x * input_y * ch_src;
337 const int32_t batch_output = output_x * output_y * ch_src;
338
339 while (batch_cnt)
340 {
341 for (int i_y = 0; i_y < output_y; i_y++)
342 {
343 for (int i_x = 0; i_x < output_x; i_x++)
344 {
345 for (int i_ch_in = 0; i_ch_in < ch_src; i_ch_in++)
346 {
347 int sum = 0;
348 int count = 0;
349 for (int k_y = i_y * stride_y - pad_y; k_y < i_y * stride_y - pad_y + kernel_y; k_y++)
350 {
351 for (int k_x = i_x * stride_x - pad_x; k_x < i_x * stride_x - pad_x + kernel_x; k_x++)
352 {
353 if (k_y >= 0 && k_x >= 0 && k_y < input_y && k_x < input_x)
354 {
355 sum += src[i_ch_in + ch_src * (k_x + k_y * input_x)];
356 count++;
357 }
358 }
359 }
360
361 // Prevent static code issue DIVIDE_BY_ZERO.
362 if (count == 0)
363 {
364 return ARM_CMSIS_NN_ARG_ERROR;
365 }
366
367 sum = sum > 0 ? (sum + count / 2) / count : (sum - count / 2) / count;
368 sum = MAX(sum, act_min);
369 sum = MIN(sum, act_max);
370
371 dst[i_ch_in + ch_src * (i_x + i_y * output_x)] = sum;
372 }
373 }
374 }
375 src += batch_input;
376 dst += batch_output;
377
378 batch_cnt--;
379 }
380
381 #endif
382 return ARM_CMSIS_NN_SUCCESS;
383 }
384
385 #endif /* ARM_MATH_MVEI */
386
387 /**
388 * @} end of Pooling group
389 */
390
