1 /*
2 * SPDX-FileCopyrightText: Copyright 2010-2024 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_convolve_1x1_s8_fast.c
22 * Description: Fast s8 version of 1x1 convolution (non-square shape)
23 *
24 * $Date: 05 November 2024
25 * $Revision: V.3.6.0
26 *
27 * Target : Arm(R) M-Profile Architecture
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
34 /**
35 * @ingroup Public
36 */
37
38 /**
39 * @addtogroup NNConv
40 * @{
41 */
42
43 /*
44 * Fast s8 version for 1x1 convolution (non-square shape)
45 *
46 * Refer header file for details.
47 *
48 */
arm_convolve_1x1_s8_fast(const cmsis_nn_context * ctx,const cmsis_nn_conv_params * conv_params,const cmsis_nn_per_channel_quant_params * quant_params,const cmsis_nn_dims * input_dims,const int8_t * input_data,const cmsis_nn_dims * filter_dims,const int8_t * filter_data,const cmsis_nn_dims * bias_dims,const int32_t * bias_data,const cmsis_nn_dims * output_dims,int8_t * output_data)49 arm_cmsis_nn_status arm_convolve_1x1_s8_fast(const cmsis_nn_context *ctx,
50 const cmsis_nn_conv_params *conv_params,
51 const cmsis_nn_per_channel_quant_params *quant_params,
52 const cmsis_nn_dims *input_dims,
53 const int8_t *input_data,
54 const cmsis_nn_dims *filter_dims,
55 const int8_t *filter_data,
56 const cmsis_nn_dims *bias_dims,
57 const int32_t *bias_data,
58 const cmsis_nn_dims *output_dims,
59 int8_t *output_data)
60 {
61 if (conv_params->padding.w != 0 || conv_params->padding.h != 0 || conv_params->stride.w != 1 ||
62 conv_params->stride.h != 1)
63 {
64 return ARM_CMSIS_NN_ARG_ERROR;
65 }
66
67 (void)filter_dims;
68 (void)bias_dims;
69
70 const int32_t rhs_cols = input_dims->c;
71 const int32_t rhs_rows = output_dims->c;
72 int32_t lhs_rows = input_dims->w * input_dims->h * input_dims->n;
73
74 #if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) && defined(__ARMCC_VERSION) && (__ARMCC_VERSION >= 6010050)
75 if (ctx->buf != NULL) /* Fall back to non buffered version if no additional memory buffer provided */
76 {
77 const int32_t batch = input_dims->n;
78 const int32_t output_h = output_dims->h;
79 const int32_t output_w = output_dims->w;
80 const int32_t input_inc = input_dims->w * rhs_cols;
81
82 for (int i_batch = 0; i_batch < batch; i_batch++)
83 {
84 const int32_t output_ch = output_dims->c;
85 const int8_t *ip = input_data;
86 int16_t *buffer_a = (int16_t *)ctx->buf;
87 int16_t *im2col_buf = (int16_t *)ctx->buf;
88 int8_t *out = output_data;
89 lhs_rows = 0;
90
91 for (int i_out_y = 0; i_out_y < output_h; i_out_y++, ip += input_inc)
92 {
93 for (int32_t k_x = 0, i_out_x = 0; i_out_x < output_w; i_out_x++, k_x += rhs_cols)
94 {
95 arm_s8_to_s16_unordered_with_offset(ip + k_x, im2col_buf, rhs_cols, conv_params->input_offset);
96 im2col_buf += rhs_cols;
97 lhs_rows++;
98 if (lhs_rows == 2)
99 {
100 out = arm_nn_mat_mult_kernel_s8_s16(filter_data,
101 buffer_a,
102 output_ch,
103 quant_params->shift,
104 quant_params->multiplier,
105 conv_params->output_offset,
106 conv_params->activation.min,
107 conv_params->activation.max,
108 rhs_cols,
109 rhs_cols,
110 bias_data,
111 out);
112 im2col_buf = buffer_a;
113 lhs_rows = 0;
114 }
115 }
116 if (out == NULL)
117 {
118 return ARM_CMSIS_NN_NO_IMPL_ERROR;
119 }
120 }
121
122 /* Handle left over columns */
123 if (lhs_rows != 0)
124 {
125 const int8_t *ker_a = filter_data;
126 for (int i = 0; i < output_ch; i++)
127 {
128 /* Load the accumulator with bias first */
129 int32_t sum = 0;
130 if (bias_data)
131 {
132 sum = bias_data[i];
133 }
134 const int16_t *ip_as_col = buffer_a;
135
136 /* 4 multiply and accumulates are done in one loop. */
137 uint16_t col_count = rhs_cols >> 2;
138 while (col_count)
139 {
140 int32_t ker_a1, ker_a2;
141 int32_t ip_b1, ip_b2;
142 ker_a = read_and_pad_reordered(ker_a, &ker_a1, &ker_a2);
143 ip_b1 = arm_nn_read_q15x2_ia(&ip_as_col);
144 sum = SMLAD(ker_a1, ip_b1, sum);
145 ip_b2 = arm_nn_read_q15x2_ia(&ip_as_col);
146 sum = SMLAD(ker_a2, ip_b2, sum);
147 col_count--;
148 }
149
150 /* Handle left over mac */
151 col_count = rhs_cols & 0x3;
152 while (col_count)
153 {
154 int8_t ker_a1 = *ker_a++;
155 int16_t ip_b1 = *ip_as_col++;
156 sum += ker_a1 * ip_b1;
157 col_count--;
158 }
159 sum = arm_nn_requantize(sum, quant_params->multiplier[i], quant_params->shift[i]);
160 sum += conv_params->output_offset;
161 sum = MAX(sum, conv_params->activation.min);
162 sum = MIN(sum, conv_params->activation.max);
163 *out++ = (int8_t)sum;
164 }
165 }
166 /* Advance to the next batch */
167 input_data += (input_dims->w * input_dims->h * rhs_cols);
168 output_data += (output_w * output_h * output_ch);
169 }
170 return ARM_CMSIS_NN_SUCCESS;
171 }
172 #else
173 (void)ctx;
174 #endif
175
176 arm_nn_mat_mult_nt_t_s8(input_data,
177 filter_data,
178 bias_data,
179 output_data,
180 quant_params->multiplier,
181 quant_params->shift,
182 lhs_rows,
183 rhs_rows,
184 rhs_cols,
185 conv_params->input_offset,
186 conv_params->output_offset,
187 conv_params->activation.min,
188 conv_params->activation.max,
189 rhs_rows,
190 rhs_cols);
191
192 /* Return to application */
193 return ARM_CMSIS_NN_SUCCESS;
194 }
195
196 /**
197 * @} end of NNConv group
198 */
199
