1 /*
2 * SPDX-FileCopyrightText: Copyright 2010-2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
3 *
4 * SPDX-License-Identifier: Apache-2.0
5 *
6 * Licensed under the Apache License, Version 2.0 (the License); you may
7 * not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 /* ----------------------------------------------------------------------
20 * Project: CMSIS NN Library
21 * Title: arm_softmax_s8.c
22 * Description: S8 softmax function
23 *
24 * $Date: 5 January 2023
25 * $Revision: V.2.2.0
26 *
27 * Target : Arm(R) M-Profile Architecture
28 *
29 * -------------------------------------------------------------------- */
30
31 #include "arm_nnfunctions.h"
32 #include "arm_nnsupportfunctions.h"
33
34 #define ACCUM_BITS 12
35
36 #if defined(ARM_MATH_MVEI) && !defined(ARM_GCC_12_2_ICE)
arm_exp_on_negative_values_mve_32x4(int32x4_t val)37 static int32x4_t arm_exp_on_negative_values_mve_32x4(int32x4_t val)
38 {
39 #define SHIFT_START (24)
40 int32_t shift = SHIFT_START;
41 int32x4_t mask;
42
43 const int32x4_t val_mod_minus_quarter =
44 vandq_s32(val, vdupq_n_s32((1 << SHIFT_START) - 1)) - vdupq_n_s32(1 << SHIFT_START);
45 const int32x4_t remainder = vsubq_s32(val_mod_minus_quarter, val);
46 const int32x4_t x = vaddq_n_s32(val_mod_minus_quarter << 5, 1 << 28);
47 const int32x4_t x2 = MUL_SAT_MVE(x, x);
48 const int32x4_t op_1 = DIV_POW2_MVE(MUL_SAT_MVE(x2, x2), 2) + MUL_SAT_MVE(x2, x);
49 const int32x4_t op_2 = x + DIV_POW2_MVE(MUL_SAT_MVE(op_1, vdupq_n_s32(715827883)) + x2, 1);
50 int32x4_t result = vdupq_n_s32(1895147668) + MUL_SAT_MVE(vdupq_n_s32(1895147668), op_2);
51
52 #define SELECT_IF_NON_ZERO(x) \
53 { \
54 mve_pred16_t p = vcmpneq_n_s32(remainder & vdupq_n_s32(1 << shift++), 0); \
55 mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p); \
56 result = SELECT_USING_MASK(mask, MUL_SAT_MVE(result, vdupq_n_s32(x)), result); \
57 }
58
59 SELECT_IF_NON_ZERO(1672461947)
60 SELECT_IF_NON_ZERO(1302514674)
61 SELECT_IF_NON_ZERO(790015084)
62 SELECT_IF_NON_ZERO(290630308)
63 SELECT_IF_NON_ZERO(39332535)
64 SELECT_IF_NON_ZERO(720401)
65 SELECT_IF_NON_ZERO(242)
66
67 #undef SELECT_IF_NON_ZERO
68
69 mve_pred16_t p = vcmpeqq_n_s32(val, 0);
70 mask = vmvnq_m_s32(vdupq_n_s32(0), vdupq_n_s32(0), p);
71
72 result = SELECT_USING_MASK(mask, vdupq_n_s32(NN_Q31_MAX), result);
73 return result;
74 }
75 #endif
76
77 /**
78 * @ingroup Public
79 */
80
81 /**
82 * @addtogroup Softmax
83 * @{
84 */
85
arm_softmax_s8(const int8_t * input,const int32_t num_rows,const int32_t row_size,const int32_t mult,const int32_t shift,const int32_t diff_min,int8_t * output)86 void arm_softmax_s8(const int8_t *input,
87 const int32_t num_rows,
88 const int32_t row_size,
89 const int32_t mult,
90 const int32_t shift,
91 const int32_t diff_min,
92 int8_t *output)
93 {
94 #if defined(ARM_MATH_MVEI) && !defined(ARM_GCC_12_2_ICE)
95
96 #define ACT_MIN ((int8_t)NN_Q7_MIN)
97 #define ACT_MAX ((int8_t)NN_Q7_MAX)
98
99 const int32_t mask = (1 << shift);
100
101 for (int i_num_rows = 0; i_num_rows < num_rows; ++i_num_rows)
102 {
103 int8_t max = ACT_MIN;
104
105 int32_t vec_count = (row_size + 15) / 16;
106 uint32_t r_count = (uint32_t)row_size;
107 for (int i = 0; i < vec_count; i++)
108 {
109 mve_pred16_t p = vctp8q(r_count);
110 const int8x16_t ip = vldrbq_z_s8(&input[i * 16], p);
111 max = vmaxvq_p_s8(max, ip, p);
112 r_count -= 16;
113 }
114
115 vec_count = row_size / 4;
116 int32_t idx = 0;
117 int32_t sum = 0;
118
119 while (vec_count)
120 {
121 int32x4_t ip = vldrbq_s32(&input[idx * 4]);
122 ip = vsubq_n_s32(ip, max);
123 mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
124 if (p != 0)
125 {
126 ip = vmulq_n_s32(ip, mask);
127
128 int32x4_t res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
129
130 res = arm_exp_on_negative_values_mve_32x4(res);
131 res = DIV_POW2_MVE(res, ACCUM_BITS);
132 res = vpselq_s32(res, vdupq_n_s32(0), p);
133 sum += vaddvq_s32(res);
134 }
135
136 vec_count--;
137 idx++;
138 }
139
140 const int32_t tail_idx = row_size & ~3;
141 for (int i = 0; i < (row_size & 3); i++)
142 {
143 const int32_t diff = input[tail_idx + i] - max;
144 if (diff >= diff_min)
145 {
146 sum += DIV_POW2(EXP_ON_NEG(MUL_SAT(diff * mask, mult)), ACCUM_BITS);
147 }
148 }
149
150 const int32_t headroom = CLZ((uint32_t)sum);
151 const int32_t bits_over_unit = ACCUM_BITS - headroom + 23;
152 const int32_t shifted_scale = ONE_OVER1((sum > 0 ? sum << headroom : 0) - (1 << 31));
153
154 vec_count = row_size / 4;
155 idx = 0;
156
157 while (vec_count)
158 {
159 int32x4_t ip = vldrbq_s32(&input[idx]);
160 ip = vsubq_n_s32(ip, max);
161
162 mve_pred16_t p = vcmpgeq_n_s32(ip, diff_min);
163
164 int32x4_t tmp_res;
165
166 if (p != 0)
167 {
168 ip = vmulq_n_s32(ip, mask);
169
170 tmp_res = MUL_SAT_MVE(ip, vdupq_n_s32(mult));
171 tmp_res = arm_exp_on_negative_values_mve_32x4(tmp_res);
172 tmp_res = MUL_SAT_MVE(vdupq_n_s32(shifted_scale), tmp_res);
173 tmp_res = DIV_POW2_MVE(tmp_res, bits_over_unit);
174 tmp_res += vdupq_n_s32(ACT_MIN);
175
176 tmp_res = vmaxq_s32(tmp_res, vdupq_n_s32(ACT_MIN));
177 tmp_res = vminq_s32(tmp_res, vdupq_n_s32(ACT_MAX));
178 tmp_res = vpselq_s32(tmp_res, vdupq_n_s32(ACT_MIN), p);
179 }
180 else
181 {
182 tmp_res = vdupq_n_s32(ACT_MIN);
183 }
184 vstrbq_s32(&output[idx], tmp_res);
185 vec_count--;
186 idx += 4;
187 }
188
189 for (int i = 0; i < (row_size & 3); i++)
190 {
191 int32_t diff = input[tail_idx + i] - max;
192 if (diff >= diff_min)
193 {
194 const int32_t res =
195 DIV_POW2(MUL_SAT(shifted_scale, EXP_ON_NEG(MUL_SAT(diff * mask, mult))), bits_over_unit) +
196 NN_Q7_MIN;
197 output[tail_idx + i] = (int8_t)CLAMP(res, (int32_t)ACT_MAX, (int32_t)ACT_MIN);
198 }
199 else
200 {
201 output[tail_idx + i] = ACT_MIN;
202 }
203 }
204
205 input += row_size;
206 output += row_size;
207 }
208 #else
209 arm_nn_softmax_common_s8(input, num_rows, row_size, mult, shift, diff_min, false, (void *)output);
210 #endif
211 }
212
213 /**
214 * @} end of Softmax group
215 */
216
