1 /*
2 * SPDX-FileCopyrightText: Copyright 2022-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_avgpool_s16.c
22 * Description: Pooling function implementations
23 *
24 * $Date: 27 November 2023
25 * $Revision: V.2.5.0
26 *
27 * Target : Arm(R) M-Profile Architecture
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
34 #if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI)
35
scale_q31_to_q15_and_clamp(const int32_t * buffer,int16_t * target,int32_t length,const int32_t count,const int act_min,const int act_max)36 static void scale_q31_to_q15_and_clamp(const int32_t *buffer,
37 int16_t *target,
38 int32_t length,
39 const int32_t count,
40 const int act_min,
41 const int act_max)
42 {
43 const int half_count = count / 2;
44
45 for (int i = 0; i < length; i++)
46 {
47 int32_t sum = buffer[i] > 0 ? (buffer[i] + half_count) : (buffer[i] - half_count);
48 sum = sum / count;
49 sum = MAX(sum, act_min);
50 sum = MIN(sum, act_max);
51
52 target[i] = (int16_t)sum;
53 }
54 }
55 #endif
56
57 /**
58 * @ingroup Public
59
60 */
61
62 /**
63 * @addtogroup Pooling
64 * @{
65 */
66
67 /*
68 * s16 average pooling function
69 *
70 * Refer to header file for details.
71 *
72 */
arm_avgpool_s16(const cmsis_nn_context * ctx,const cmsis_nn_pool_params * pool_params,const cmsis_nn_dims * input_dims,const int16_t * src,const cmsis_nn_dims * filter_dims,const cmsis_nn_dims * output_dims,int16_t * dst)73 arm_cmsis_nn_status arm_avgpool_s16(const cmsis_nn_context *ctx,
74 const cmsis_nn_pool_params *pool_params,
75 const cmsis_nn_dims *input_dims,
76 const int16_t *src,
77 const cmsis_nn_dims *filter_dims,
78 const cmsis_nn_dims *output_dims,
79 int16_t *dst)
80 {
81 const int32_t input_y = input_dims->h;
82 const int32_t input_x = input_dims->w;
83 const int32_t output_y = output_dims->h;
84 const int32_t output_x = output_dims->w;
85 const int32_t stride_y = pool_params->stride.h;
86 const int32_t stride_x = pool_params->stride.w;
87 const int32_t kernel_y = filter_dims->h;
88 const int32_t kernel_x = filter_dims->w;
89 const int32_t pad_y = pool_params->padding.h;
90 const int32_t pad_x = pool_params->padding.w;
91 const int32_t act_min = pool_params->activation.min;
92 const int32_t act_max = pool_params->activation.max;
93 const int32_t ch_src = input_dims->c;
94 const int32_t batch_input = input_x * input_y * ch_src;
95 int32_t batch_cnt = input_dims->n;
96
97 if (batch_cnt < 1)
98 {
99 return ARM_CMSIS_NN_ARG_ERROR;
100 }
101
102 #if defined(ARM_MATH_MVEI)
103 (void)ctx;
104
105 const int32_t batch_output = output_x * output_y * ch_src;
106
107 while (batch_cnt)
108 {
109 for (int i_y = 0; i_y < output_y; i_y++)
110 {
111 for (int i_x = 0; i_x < output_x; i_x++)
112 {
113 const int32_t k_y_start = MAX(0, i_y * stride_y - pad_y);
114 const int32_t k_y_end = MIN(i_y * stride_y - pad_y + kernel_y, input_y);
115
116 const int32_t k_x_start = MAX(0, i_x * stride_x - pad_x);
117 const int32_t k_x_end = MIN(i_x * stride_x - pad_x + kernel_x, input_x);
118
119 const int16_t *src_base = src;
120 int16_t *out = &dst[ch_src * (i_x + i_y * output_x)];
121
122 int32_t ch_count = (ch_src + 7) / 8;
123 int32_t channels = ch_src;
124
125 while (ch_count > 0)
126 {
127 int32_t count = 0;
128
129 int32x4_t sum_1 = vdupq_n_s32(0);
130 int32x4_t sum_2 = vdupq_n_s32(0);
131 // Load store tail predicate
132 const mve_pred16_t ld_st_p = vctp16q(channels);
133 channels -= 8;
134
135 for (int k_y = k_y_start; k_y < k_y_end; k_y++)
136 {
137 for (int k_x = k_x_start; k_x < k_x_end; k_x++)
138 {
139 const int16_t *src_inner = src_base + (ch_src * (k_x + k_y * input_x));
140 const int16x8_t temp = vldrhq_z_s16(src_inner, ld_st_p);
141
142 const int32x4_t temp_lo = vmovlbq_s16(temp);
143 const int32x4_t temp_hi = vmovltq_s16(temp);
144
145 sum_1 = vaddq_s32(sum_1, temp_lo);
146 sum_2 = vaddq_s32(sum_2, temp_hi);
147
148 count++;
149 }
150 }
151
152 // Prevent static code issue DIVIDE_BY_ZERO.
153 if (count == 0)
154 {
155 return ARM_CMSIS_NN_ARG_ERROR;
156 }
157
158 // Perform the following operation
159 // sum = sum > 0 ? (sum + count / 2) / count : (sum - count / 2) / count;
160 const int32_t half_count = count / 2;
161 // Predicate for 'sum > 0' operation
162 mve_pred16_t p = vcmpgtq_n_s32(sum_1, 0);
163 sum_1 = vaddq_m_n_s32(sum_1, sum_1, half_count, p);
164 sum_1 = vsubq_m_n_s32(sum_1, sum_1, half_count, ~p);
165
166 p = vcmpgtq_n_s32(sum_2, 0);
167 sum_2 = vaddq_m_n_s32(sum_2, sum_2, half_count, p);
168 sum_2 = vsubq_m_n_s32(sum_2, sum_2, half_count, ~p);
169
170 for (int i = 0; i < 4; i++)
171 {
172 sum_1[i] = sum_1[i] / count;
173 sum_2[i] = sum_2[i] / count;
174 }
175
176 sum_1 = vmaxq_s32(sum_1, vdupq_n_s32(act_min));
177 sum_1 = vminq_s32(sum_1, vdupq_n_s32(act_max));
178
179 sum_2 = vmaxq_s32(sum_2, vdupq_n_s32(act_min));
180 sum_2 = vminq_s32(sum_2, vdupq_n_s32(act_max));
181
182 int16x8_t temp = vdupq_n_s16(0);
183 temp = vmovnbq_s32(temp, sum_1);
184 temp = vmovntq_s32(temp, sum_2);
185
186 vstrhq_p_s16(out, temp, ld_st_p);
187
188 out += 8;
189 ch_count--;
190 src_base += 8;
191 }
192 }
193 }
194 src += batch_input;
195 dst += batch_output;
196
197 batch_cnt--;
198 }
199
200 #elif defined(ARM_MATH_DSP)
201 /* Run the following code for CPU's with DSP extension
202 */
203 int32_t *buffer = (int32_t *)ctx->buf;
204
205 if (buffer == NULL)
206 {
207 return ARM_CMSIS_NN_ARG_ERROR;
208 }
209
210 while (batch_cnt)
211 {
212
213 for (int i_y = 0, idx_y = -pad_y; i_y < output_y; idx_y += stride_y, i_y++)
214 {
215 for (int i_x = 0, idx_x = -pad_x; i_x < output_x; idx_x += stride_x, i_x++)
216 {
217 /* Condition for kernel start dimension:
218 (base_idx_<x,y> + kernel_<x,y>_start) >= 0 */
219 const int32_t kernel_y_start = MAX(0, -idx_y);
220 const int32_t kernel_x_start = MAX(0, -idx_x);
221
222 /* Condition for kernel end dimension:
223 (base_idx_<x,y> + kernel_<x,y>_end) < dim_src_<width,height> */
224 const int32_t kernel_y_end = MIN(kernel_y, input_y - idx_y);
225 const int32_t kernel_x_end = MIN(kernel_x, input_x - idx_x);
226
227 int count = 0;
228
229 for (int k_y = kernel_y_start; k_y < kernel_y_end; k_y++)
230 {
231 for (int k_x = kernel_x_start; k_x < kernel_x_end; k_x++)
232 {
233 const int16_t *start = src + ch_src * (k_x + idx_x + (k_y + idx_y) * input_x);
234
235 if (count == 0)
236 {
237 for (int i = 0; i < ch_src; i++)
238 {
239 buffer[i] = start[i];
240 }
241 }
242 else
243 {
244 for (int i = 0; i < ch_src; i++)
245 {
246 buffer[i] = QADD(start[i], buffer[i]);
247 }
248 }
249 count++;
250 }
251 }
252
253 // Prevent static code issue DIVIDE_BY_ZERO.
254 if (count == 0)
255 {
256 return ARM_CMSIS_NN_ARG_ERROR;
257 }
258
259 scale_q31_to_q15_and_clamp(buffer, dst, ch_src, count, act_min, act_max);
260 dst += ch_src;
261 }
262 }
263 src += batch_input;
264
265 batch_cnt--;
266 }
267
268 #else
269 /* Reference C code adapted from CMSIS-NN arm_avgpool_s8.c.
270 */
271 const int32_t batch_output = output_x * output_y * ch_src;
272 (void)ctx;
273
274 while (batch_cnt)
275 {
276 for (int i_y = 0, base_idx_y = -pad_y; i_y < output_y; base_idx_y += stride_y, i_y++)
277 {
278 for (int i_x = 0, base_idx_x = -pad_x; i_x < output_x; base_idx_x += stride_x, i_x++)
279 {
280 /* Condition for kernel start dimension: (base_idx_<x,y> + kernel_<x,y>_start) >= 0 */
281 const int32_t ker_y_start = MAX(0, -base_idx_y);
282 const int32_t ker_x_start = MAX(0, -base_idx_x);
283
284 /* Condition for kernel end dimension: (base_idx_<x,y> + kernel_<x,y>_end) < dim_src_<width,height> */
285 const int32_t kernel_y_end = MIN(kernel_y, input_y - base_idx_y);
286 const int32_t kernel_x_end = MIN(kernel_x, input_x - base_idx_x);
287
288 for (int i_ch_in = 0; i_ch_in < ch_src; i_ch_in++)
289 {
290 int sum = 0;
291 int count = 0;
292
293 for (int k_y = ker_y_start; k_y < kernel_y_end; k_y++)
294 {
295 for (int k_x = ker_x_start; k_x < kernel_x_end; k_x++)
296 {
297 sum += src[i_ch_in + ch_src * (k_x + base_idx_x + (k_y + base_idx_y) * input_x)];
298 count++;
299 }
300 }
301
302 // Prevent static code issue DIVIDE_BY_ZERO.
303 if (count == 0)
304 {
305 return ARM_CMSIS_NN_ARG_ERROR;
306 }
307
308 sum = sum > 0 ? (sum + count / 2) / count : (sum - count / 2) / count;
309 sum = MAX(sum, act_min);
310 sum = MIN(sum, act_max);
311
312 dst[i_ch_in + ch_src * (i_x + i_y * output_x)] = sum;
313 }
314 }
315 }
316 src += batch_input;
317 dst += batch_output;
318
319 batch_cnt--;
320 }
321 #endif
322
323 return ARM_CMSIS_NN_SUCCESS;
324 }
325
326 /**
327 * @} end of Pooling group
328 */
329
