1 /*
2 * SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_max_pool_s8.c
22 * Description: Pooling function implementations
23 *
24 * $Date: 27 November 2023
25 * $Revision: V.3.1.0
26 *
27 * Target Processor: Cortex-M CPUs
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
compare_and_replace_if_larger_q7(int8_t * base,const int8_t * target,int32_t length)34 static void compare_and_replace_if_larger_q7(int8_t *base, const int8_t *target, int32_t length)
35 {
36 #if defined(ARM_MATH_MVEI)
37 int32_t loop_count = (length + 15) / 16;
38 for (int i = 0; i < loop_count; i++)
39 {
40 mve_pred16_t p = vctp8q((uint32_t)length);
41 const int8x16_t op_1 = vldrbq_z_s8(base, p);
42 const int8x16_t op_2 = vldrbq_z_s8(target, p);
43 const int8x16_t max = vmaxq_x_s8(op_1, op_2, p);
44 vstrbq_p_s8(base, max, p);
45 base += 16;
46 target += 16;
47 length -= 16;
48 }
49 #else
50 int8_t *dst = base;
51 const int8_t *src = target;
52 union arm_nnword ref_max;
53 union arm_nnword comp_max;
54 int32_t cnt = length >> 2;
55
56 while (cnt > 0l)
57 {
58 ref_max.word = arm_nn_read_s8x4(dst);
59 comp_max.word = arm_nn_read_s8x4_ia(&src);
60
61 if (comp_max.bytes[0] > ref_max.bytes[0])
62 {
63 ref_max.bytes[0] = comp_max.bytes[0];
64 }
65 if (comp_max.bytes[1] > ref_max.bytes[1])
66 {
67 ref_max.bytes[1] = comp_max.bytes[1];
68 }
69 if (comp_max.bytes[2] > ref_max.bytes[2])
70 {
71 ref_max.bytes[2] = comp_max.bytes[2];
72 }
73 if (comp_max.bytes[3] > ref_max.bytes[3])
74 {
75 ref_max.bytes[3] = comp_max.bytes[3];
76 }
77
78 arm_nn_write_s8x4_ia(&dst, ref_max.word);
79
80 cnt--;
81 }
82
83 cnt = length & 0x3;
84 while (cnt > 0l)
85 {
86 if (*src > *dst)
87 {
88 *dst = *src;
89 }
90 dst++;
91 src++;
92 cnt--;
93 }
94 #endif
95 }
96
clamp_output(int8_t * source,int32_t length,const int32_t act_min,const int32_t act_max)97 static void clamp_output(int8_t *source, int32_t length, const int32_t act_min, const int32_t act_max)
98 {
99 #if defined(ARM_MATH_MVEI)
100 int32_t loop_count = (length + 15) / 16;
101 const int8x16_t vmin = vdupq_n_s8((int8_t)act_min);
102 const int8x16_t vmax = vdupq_n_s8((int8_t)act_max);
103
104 for (int i = 0; i < loop_count; i++)
105 {
106 mve_pred16_t p = vctp8q((uint32_t)length);
107 length -= 16;
108 const int8x16_t src = vldrbq_z_s8(source, p);
109 int8x16_t res = vmaxq_x_s8(src, vmin, p);
110 res = vminq_x_s8(res, vmax, p);
111 vstrbq_p_s8(source, res, p);
112 source += 16;
113 }
114 #else
115 union arm_nnword in;
116 int32_t cnt = length >> 2;
117
118 while (cnt > 0l)
119 {
120 in.word = arm_nn_read_s8x4(source);
121
122 in.bytes[0] = MAX(in.bytes[0], act_min);
123 in.bytes[0] = MIN(in.bytes[0], act_max);
124 in.bytes[1] = MAX(in.bytes[1], act_min);
125 in.bytes[1] = MIN(in.bytes[1], act_max);
126 in.bytes[2] = MAX(in.bytes[2], act_min);
127 in.bytes[2] = MIN(in.bytes[2], act_max);
128 in.bytes[3] = MAX(in.bytes[3], act_min);
129 in.bytes[3] = MIN(in.bytes[3], act_max);
130
131 arm_nn_write_s8x4_ia(&source, in.word);
132 cnt--;
133 }
134
135 cnt = length & 0x3;
136 while (cnt > 0l)
137 {
138 int32_t comp = *source;
139 comp = MAX(comp, act_min);
140 comp = MIN(comp, act_max);
141 *source++ = (int8_t)comp;
142 cnt--;
143 }
144 #endif
145 }
146
147 /**
148 * @ingroup Public
149 */
150
151 /**
152 * @addtogroup Pooling
153 * @{
154 */
155
156 /*
157 * Optimized s8 max pooling function
158 *
159 * Refer to header file for details.
160 *
161 */
arm_max_pool_s8(const cmsis_nn_context * ctx,const cmsis_nn_pool_params * pool_params,const cmsis_nn_dims * input_dims,const int8_t * src,const cmsis_nn_dims * filter_dims,const cmsis_nn_dims * output_dims,int8_t * dst)162 arm_cmsis_nn_status arm_max_pool_s8(const cmsis_nn_context *ctx,
163 const cmsis_nn_pool_params *pool_params,
164 const cmsis_nn_dims *input_dims,
165 const int8_t *src,
166 const cmsis_nn_dims *filter_dims,
167 const cmsis_nn_dims *output_dims,
168 int8_t *dst)
169 {
170 (void)ctx;
171 const int32_t input_y = input_dims->h;
172 const int32_t input_x = input_dims->w;
173 const int32_t output_y = output_dims->h;
174 const int32_t output_x = output_dims->w;
175 const int32_t stride_y = pool_params->stride.h;
176 const int32_t stride_x = pool_params->stride.w;
177 const int32_t kernel_y = filter_dims->h;
178 const int32_t kernel_x = filter_dims->w;
179 const int32_t pad_y = pool_params->padding.h;
180 const int32_t pad_x = pool_params->padding.w;
181 const int32_t act_min = pool_params->activation.min;
182 const int32_t act_max = pool_params->activation.max;
183 const int32_t channel_in = input_dims->c;
184 const int32_t batch_size = input_x * input_y * channel_in;
185 int32_t batch_cnt = input_dims->n;
186
187 if (batch_cnt < 1)
188 {
189 return ARM_CMSIS_NN_ARG_ERROR;
190 }
191
192 while (batch_cnt)
193 {
194 int8_t *dst_base = dst;
195
196 for (int i_y = 0, base_idx_y = -pad_y; i_y < output_y; base_idx_y += stride_y, i_y++)
197 {
198 for (int i_x = 0, base_idx_x = -pad_x; i_x < output_x; base_idx_x += stride_x, i_x++)
199 {
200 /* Condition for kernel start dimension: (base_idx_<x,y> + kernel_<x,y>_start) >= 0 */
201 const int32_t ker_y_start = MAX(0, -base_idx_y);
202 const int32_t ker_x_start = MAX(0, -base_idx_x);
203
204 /* Condition for kernel end dimension: (base_idx_<x,y> + kernel_<x,y>_end) < dim_src_<width,height> */
205 const int32_t kernel_y_end = MIN(kernel_y, input_y - base_idx_y);
206 const int32_t kernel_x_end = MIN(kernel_x, input_x - base_idx_x);
207
208 int count = 0;
209
210 for (int k_y = ker_y_start; k_y < kernel_y_end; k_y++)
211 {
212 for (int k_x = ker_x_start; k_x < kernel_x_end; k_x++)
213 {
214 const int8_t *start = src + channel_in * (k_x + base_idx_x + (k_y + base_idx_y) * input_x);
215
216 if (count == 0)
217 {
218 arm_memcpy_s8(dst, start, channel_in);
219 count++;
220 }
221 else
222 {
223 compare_and_replace_if_larger_q7(dst, start, channel_in);
224 }
225 }
226 }
227 /* 'count' is expected to be non-zero here. */
228 dst += channel_in;
229 }
230 }
231
232 clamp_output(dst_base, output_x * output_y * channel_in, act_min, act_max);
233
234 src += batch_size;
235 batch_cnt--;
236 }
237
238 return ARM_CMSIS_NN_SUCCESS;
239 }
240
241 /**
242 * @} end of Pooling group
243 */
244
