1 /* ----------------------------------------------------------------------
2 * Project: CMSIS DSP Library
3 * Title: arm_correlate_fast_q31.c
4 * Description: Fast Q31 Correlation
5 *
6 * $Date: 23 April 2021
7 * $Revision: V1.9.0
8 *
9 * Target Processor: Cortex-M and Cortex-A cores
10 * -------------------------------------------------------------------- */
11 /*
12 * Copyright (C) 2010-2021 ARM Limited or its affiliates. All rights reserved.
13 *
14 * SPDX-License-Identifier: Apache-2.0
15 *
16 * Licensed under the Apache License, Version 2.0 (the License); you may
17 * not use this file except in compliance with the License.
18 * You may obtain a copy of the License at
19 *
20 * www.apache.org/licenses/LICENSE-2.0
21 *
22 * Unless required by applicable law or agreed to in writing, software
23 * distributed under the License is distributed on an AS IS BASIS, WITHOUT
24 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 * See the License for the specific language governing permissions and
26 * limitations under the License.
27 */
28
29 #include "dsp/filtering_functions.h"
30
31 /**
32 @ingroup groupFilters
33 */
34
35 /**
36 @addtogroup Corr
37 @{
38 */
39
40 /**
41 @brief Correlation of Q31 sequences (fast version).
42 @param[in] pSrcA points to the first input sequence
43 @param[in] srcALen length of the first input sequence
44 @param[in] pSrcB points to the second input sequence
45 @param[in] srcBLen length of the second input sequence
46 @param[out] pDst points to the location where the output result is written. Length 2 * max(srcALen, srcBLen) - 1.
47
48 @par Scaling and Overflow Behavior
49 This function is optimized for speed at the expense of fixed-point precision and overflow protection.
50 The result of each 1.31 x 1.31 multiplication is truncated to 2.30 format.
51 These intermediate results are accumulated in a 32-bit register in 2.30 format.
52 Finally, the accumulator is saturated and converted to a 1.31 result.
53 @par
54 The fast version has the same overflow behavior as the standard version but provides less precision since it discards the low 32 bits of each multiplication result.
55 In order to avoid overflows completely the input signals must be scaled down.
56 The input signals should be scaled down to avoid intermediate overflows.
57 Scale down one of the inputs by 1/min(srcALen, srcBLen)to avoid overflows since a
58 maximum of min(srcALen, srcBLen) number of additions is carried internally.
59
60 @remark
61 Refer to \ref arm_correlate_q31() for a slower implementation of this function which uses 64-bit accumulation to provide higher precision.
62 */
63
arm_correlate_fast_q31(const q31_t * pSrcA,uint32_t srcALen,const q31_t * pSrcB,uint32_t srcBLen,q31_t * pDst)64 void arm_correlate_fast_q31(
65 const q31_t * pSrcA,
66 uint32_t srcALen,
67 const q31_t * pSrcB,
68 uint32_t srcBLen,
69 q31_t * pDst)
70 {
71 const q31_t *pIn1; /* InputA pointer */
72 const q31_t *pIn2; /* InputB pointer */
73 q31_t *pOut = pDst; /* Output pointer */
74 const q31_t *px; /* Intermediate inputA pointer */
75 const q31_t *py; /* Intermediate inputB pointer */
76 const q31_t *pSrc1; /* Intermediate pointers */
77 q31_t sum, acc0, acc1, acc2, acc3; /* Accumulators */
78 q31_t x0, x1, x2, x3, c0; /* Temporary variables for holding input and coefficient values */
79 uint32_t blockSize1, blockSize2, blockSize3; /* Loop counters */
80 uint32_t j, k, count, blkCnt; /* Loop counters */
81 uint32_t outBlockSize;
82 int32_t inc = 1; /* Destination address modifier */
83
84 /* The algorithm implementation is based on the lengths of the inputs. */
85 /* srcB is always made to slide across srcA. */
86 /* So srcBLen is always considered as shorter or equal to srcALen */
87 if (srcALen >= srcBLen)
88 {
89 /* Initialization of inputA pointer */
90 pIn1 = pSrcA;
91
92 /* Initialization of inputB pointer */
93 pIn2 = pSrcB;
94
95 /* Number of output samples is calculated */
96 outBlockSize = (2U * srcALen) - 1U;
97
98 /* When srcALen > srcBLen, zero padding is done to srcB
99 * to make their lengths equal.
100 * Instead, (outBlockSize - (srcALen + srcBLen - 1))
101 * number of output samples are made zero */
102 j = outBlockSize - (srcALen + (srcBLen - 1U));
103
104 /* Updating the pointer position to non zero value */
105 pOut += j;
106
107 }
108 else
109 {
110 /* Initialization of inputA pointer */
111 pIn1 = pSrcB;
112
113 /* Initialization of inputB pointer */
114 pIn2 = pSrcA;
115
116 /* srcBLen is always considered as shorter or equal to srcALen */
117 j = srcBLen;
118 srcBLen = srcALen;
119 srcALen = j;
120
121 /* CORR(x, y) = Reverse order(CORR(y, x)) */
122 /* Hence set the destination pointer to point to the last output sample */
123 pOut = pDst + ((srcALen + srcBLen) - 2U);
124
125 /* Destination address modifier is set to -1 */
126 inc = -1;
127
128 }
129
130 /* The function is internally
131 * divided into three stages according to the number of multiplications that has to be
132 * taken place between inputA samples and inputB samples. In the first stage of the
133 * algorithm, the multiplications increase by one for every iteration.
134 * In the second stage of the algorithm, srcBLen number of multiplications are done.
135 * In the third stage of the algorithm, the multiplications decrease by one
136 * for every iteration. */
137
138 /* The algorithm is implemented in three stages.
139 The loop counters of each stage is initiated here. */
140 blockSize1 = srcBLen - 1U;
141 blockSize2 = srcALen - (srcBLen - 1U);
142 blockSize3 = blockSize1;
143
144 /* --------------------------
145 * Initializations of stage1
146 * -------------------------*/
147
148 /* sum = x[0] * y[srcBlen - 1]
149 * sum = x[0] * y[srcBlen - 2] + x[1] * y[srcBlen - 1]
150 * ....
151 * sum = x[0] * y[0] + x[1] * y[1] +...+ x[srcBLen - 1] * y[srcBLen - 1]
152 */
153
154 /* In this stage the MAC operations are increased by 1 for every iteration.
155 The count variable holds the number of MAC operations performed */
156 count = 1U;
157
158 /* Working pointer of inputA */
159 px = pIn1;
160
161 /* Working pointer of inputB */
162 pSrc1 = pIn2 + (srcBLen - 1U);
163 py = pSrc1;
164
165 /* ------------------------
166 * Stage1 process
167 * ----------------------*/
168
169 /* The first stage starts here */
170 while (blockSize1 > 0U)
171 {
172 /* Accumulator is made zero for every iteration */
173 sum = 0;
174
175 /* Apply loop unrolling and compute 4 MACs simultaneously. */
176 k = count >> 2U;
177
178 /* First part of the processing with loop unrolling. Compute 4 MACs at a time.
179 ** a second loop below computes MACs for the remaining 1 to 3 samples. */
180 while (k > 0U)
181 {
182 /* x[0] * y[srcBLen - 4] */
183 sum = (q31_t) ((((q63_t) sum << 32) +
184 ((q63_t) *px++ * (*py++))) >> 32);
185
186 /* x[1] * y[srcBLen - 3] */
187 sum = (q31_t) ((((q63_t) sum << 32) +
188 ((q63_t) *px++ * (*py++))) >> 32);
189
190 /* x[2] * y[srcBLen - 2] */
191 sum = (q31_t) ((((q63_t) sum << 32) +
192 ((q63_t) *px++ * (*py++))) >> 32);
193
194 /* x[3] * y[srcBLen - 1] */
195 sum = (q31_t) ((((q63_t) sum << 32) +
196 ((q63_t) *px++ * (*py++))) >> 32);
197
198 /* Decrement loop counter */
199 k--;
200 }
201
202 /* If the count is not a multiple of 4, compute any remaining MACs here.
203 ** No loop unrolling is used. */
204 k = count % 0x4U;
205
206 while (k > 0U)
207 {
208 /* Perform the multiply-accumulate */
209 /* x[0] * y[srcBLen - 1] */
210 sum = (q31_t) ((((q63_t) sum << 32) +
211 ((q63_t) *px++ * (*py++))) >> 32);
212
213 /* Decrement loop counter */
214 k--;
215 }
216
217 /* Store the result in the accumulator in the destination buffer. */
218 *pOut = sum << 1;
219 /* Destination pointer is updated according to the address modifier, inc */
220 pOut += inc;
221
222 /* Update the inputA and inputB pointers for next MAC calculation */
223 py = pSrc1 - count;
224 px = pIn1;
225
226 /* Increment MAC count */
227 count++;
228
229 /* Decrement loop counter */
230 blockSize1--;
231 }
232
233 /* --------------------------
234 * Initializations of stage2
235 * ------------------------*/
236
237 /* sum = x[0] * y[0] + x[1] * y[1] +...+ x[srcBLen-1] * y[srcBLen-1]
238 * sum = x[1] * y[0] + x[2] * y[1] +...+ x[srcBLen] * y[srcBLen-1]
239 * ....
240 * sum = x[srcALen-srcBLen-2] * y[0] + x[srcALen-srcBLen-1] * y[1] +...+ x[srcALen-1] * y[srcBLen-1]
241 */
242
243 /* Working pointer of inputA */
244 px = pIn1;
245
246 /* Working pointer of inputB */
247 py = pIn2;
248
249 /* count is index by which the pointer pIn1 to be incremented */
250 count = 0U;
251
252 /* -------------------
253 * Stage2 process
254 * ------------------*/
255
256 /* Stage2 depends on srcBLen as in this stage srcBLen number of MACS are performed.
257 * So, to loop unroll over blockSize2,
258 * srcBLen should be greater than or equal to 4 */
259 if (srcBLen >= 4U)
260 {
261 /* Loop unroll over blockSize2, by 4 */
262 blkCnt = blockSize2 >> 2U;
263
264 while (blkCnt > 0U)
265 {
266 /* Set all accumulators to zero */
267 acc0 = 0;
268 acc1 = 0;
269 acc2 = 0;
270 acc3 = 0;
271
272 /* read x[0], x[1], x[2] samples */
273 x0 = *px++;
274 x1 = *px++;
275 x2 = *px++;
276
277 /* Apply loop unrolling and compute 4 MACs simultaneously. */
278 k = srcBLen >> 2U;
279
280 /* First part of the processing with loop unrolling. Compute 4 MACs at a time.
281 ** a second loop below computes MACs for the remaining 1 to 3 samples. */
282 do
283 {
284 /* Read y[0] sample */
285 c0 = *py++;
286 /* Read x[3] sample */
287 x3 = *px++;
288
289 /* Perform the multiply-accumulate */
290 /* acc0 += x[0] * y[0] */
291 acc0 = (q31_t) ((((q63_t) acc0 << 32) + ((q63_t) x0 * c0)) >> 32);
292 /* acc1 += x[1] * y[0] */
293 acc1 = (q31_t) ((((q63_t) acc1 << 32) + ((q63_t) x1 * c0)) >> 32);
294 /* acc2 += x[2] * y[0] */
295 acc2 = (q31_t) ((((q63_t) acc2 << 32) + ((q63_t) x2 * c0)) >> 32);
296 /* acc3 += x[3] * y[0] */
297 acc3 = (q31_t) ((((q63_t) acc3 << 32) + ((q63_t) x3 * c0)) >> 32);
298
299
300 /* Read y[1] sample */
301 c0 = *py++;
302 /* Read x[4] sample */
303 x0 = *px++;
304
305 /* Perform the multiply-accumulate */
306 /* acc0 += x[1] * y[1] */
307 acc0 = (q31_t) ((((q63_t) acc0 << 32) + ((q63_t) x1 * c0)) >> 32);
308 /* acc1 += x[2] * y[1] */
309 acc1 = (q31_t) ((((q63_t) acc1 << 32) + ((q63_t) x2 * c0)) >> 32);
310 /* acc2 += x[3] * y[1] */
311 acc2 = (q31_t) ((((q63_t) acc2 << 32) + ((q63_t) x3 * c0)) >> 32);
312 /* acc3 += x[4] * y[1] */
313 acc3 = (q31_t) ((((q63_t) acc3 << 32) + ((q63_t) x0 * c0)) >> 32);
314
315
316 /* Read y[2] sample */
317 c0 = *py++;
318 /* Read x[5] sample */
319 x1 = *px++;
320
321 /* Perform the multiply-accumulates */
322 /* acc0 += x[2] * y[2] */
323 acc0 = (q31_t) ((((q63_t) acc0 << 32) + ((q63_t) x2 * c0)) >> 32);
324 /* acc1 += x[3] * y[2] */
325 acc1 = (q31_t) ((((q63_t) acc1 << 32) + ((q63_t) x3 * c0)) >> 32);
326 /* acc2 += x[4] * y[2] */
327 acc2 = (q31_t) ((((q63_t) acc2 << 32) + ((q63_t) x0 * c0)) >> 32);
328 /* acc3 += x[5] * y[2] */
329 acc3 = (q31_t) ((((q63_t) acc3 << 32) + ((q63_t) x1 * c0)) >> 32);
330
331
332 /* Read y[3] sample */
333 c0 = *py++;
334 /* Read x[6] sample */
335 x2 = *px++;
336
337 /* Perform the multiply-accumulates */
338 /* acc0 += x[3] * y[3] */
339 acc0 = (q31_t) ((((q63_t) acc0 << 32) + ((q63_t) x3 * c0)) >> 32);
340 /* acc1 += x[4] * y[3] */
341 acc1 = (q31_t) ((((q63_t) acc1 << 32) + ((q63_t) x0 * c0)) >> 32);
342 /* acc2 += x[5] * y[3] */
343 acc2 = (q31_t) ((((q63_t) acc2 << 32) + ((q63_t) x1 * c0)) >> 32);
344 /* acc3 += x[6] * y[3] */
345 acc3 = (q31_t) ((((q63_t) acc3 << 32) + ((q63_t) x2 * c0)) >> 32);
346
347
348 } while (--k);
349
350 /* If the srcBLen is not a multiple of 4, compute any remaining MACs here.
351 ** No loop unrolling is used. */
352 k = srcBLen % 0x4U;
353
354 while (k > 0U)
355 {
356 /* Read y[4] sample */
357 c0 = *py++;
358 /* Read x[7] sample */
359 x3 = *px++;
360
361 /* Perform the multiply-accumulates */
362 /* acc0 += x[4] * y[4] */
363 acc0 = (q31_t) ((((q63_t) acc0 << 32) + ((q63_t) x0 * c0)) >> 32);
364 /* acc1 += x[5] * y[4] */
365 acc1 = (q31_t) ((((q63_t) acc1 << 32) + ((q63_t) x1 * c0)) >> 32);
366 /* acc2 += x[6] * y[4] */
367 acc2 = (q31_t) ((((q63_t) acc2 << 32) + ((q63_t) x2 * c0)) >> 32);
368 /* acc3 += x[7] * y[4] */
369 acc3 = (q31_t) ((((q63_t) acc3 << 32) + ((q63_t) x3 * c0)) >> 32);
370
371 /* Reuse the present samples for the next MAC */
372 x0 = x1;
373 x1 = x2;
374 x2 = x3;
375
376 /* Decrement loop counter */
377 k--;
378 }
379
380 /* Store the result in the accumulator in the destination buffer. */
381 *pOut = (q31_t) (acc0 << 1);
382 /* Destination pointer is updated according to the address modifier, inc */
383 pOut += inc;
384
385 *pOut = (q31_t) (acc1 << 1);
386 pOut += inc;
387
388 *pOut = (q31_t) (acc2 << 1);
389 pOut += inc;
390
391 *pOut = (q31_t) (acc3 << 1);
392 pOut += inc;
393
394 /* Increment the pointer pIn1 index, count by 4 */
395 count += 4U;
396
397 /* Update the inputA and inputB pointers for next MAC calculation */
398 px = pIn1 + count;
399 py = pIn2;
400
401 /* Decrement loop counter */
402 blkCnt--;
403 }
404
405 /* If the blockSize2 is not a multiple of 4, compute any remaining output samples here.
406 ** No loop unrolling is used. */
407 blkCnt = blockSize2 % 0x4U;
408
409 while (blkCnt > 0U)
410 {
411 /* Accumulator is made zero for every iteration */
412 sum = 0;
413
414 /* Apply loop unrolling and compute 4 MACs simultaneously. */
415 k = srcBLen >> 2U;
416
417 /* First part of the processing with loop unrolling. Compute 4 MACs at a time.
418 ** a second loop below computes MACs for the remaining 1 to 3 samples. */
419 while (k > 0U)
420 {
421 /* Perform the multiply-accumulates */
422 sum = (q31_t) ((((q63_t) sum << 32) +
423 ((q63_t) *px++ * (*py++))) >> 32);
424 sum = (q31_t) ((((q63_t) sum << 32) +
425 ((q63_t) *px++ * (*py++))) >> 32);
426 sum = (q31_t) ((((q63_t) sum << 32) +
427 ((q63_t) *px++ * (*py++))) >> 32);
428 sum = (q31_t) ((((q63_t) sum << 32) +
429 ((q63_t) *px++ * (*py++))) >> 32);
430
431 /* Decrement loop counter */
432 k--;
433 }
434
435 /* If the srcBLen is not a multiple of 4, compute any remaining MACs here.
436 ** No loop unrolling is used. */
437 k = srcBLen % 0x4U;
438
439 while (k > 0U)
440 {
441 /* Perform the multiply-accumulate */
442 sum = (q31_t) ((((q63_t) sum << 32) +
443 ((q63_t) *px++ * (*py++))) >> 32);
444
445 /* Decrement loop counter */
446 k--;
447 }
448
449 /* Store the result in the accumulator in the destination buffer. */
450 *pOut = sum << 1;
451 /* Destination pointer is updated according to the address modifier, inc */
452 pOut += inc;
453
454 /* Increment MAC count */
455 count++;
456
457 /* Update the inputA and inputB pointers for next MAC calculation */
458 px = pIn1 + count;
459 py = pIn2;
460
461 /* Decrement loop counter */
462 blkCnt--;
463 }
464 }
465 else
466 {
467 /* If the srcBLen is not a multiple of 4,
468 * the blockSize2 loop cannot be unrolled by 4 */
469 blkCnt = blockSize2;
470
471 while (blkCnt > 0U)
472 {
473 /* Accumulator is made zero for every iteration */
474 sum = 0;
475
476 /* srcBLen number of MACS should be performed */
477 k = srcBLen;
478
479 while (k > 0U)
480 {
481 /* Perform the multiply-accumulate */
482 sum = (q31_t) ((((q63_t) sum << 32) +
483 ((q63_t) *px++ * (*py++))) >> 32);
484
485 /* Decrement loop counter */
486 k--;
487 }
488
489 /* Store the result in the accumulator in the destination buffer. */
490 *pOut = sum << 1;
491 /* Destination pointer is updated according to the address modifier, inc */
492 pOut += inc;
493
494 /* Increment MAC count */
495 count++;
496
497 /* Update the inputA and inputB pointers for next MAC calculation */
498 px = pIn1 + count;
499 py = pIn2;
500
501 /* Decrement loop counter */
502 blkCnt--;
503 }
504 }
505
506
507 /* --------------------------
508 * Initializations of stage3
509 * -------------------------*/
510
511 /* sum += x[srcALen-srcBLen+1] * y[0] + x[srcALen-srcBLen+2] * y[1] +...+ x[srcALen-1] * y[srcBLen-1]
512 * sum += x[srcALen-srcBLen+2] * y[0] + x[srcALen-srcBLen+3] * y[1] +...+ x[srcALen-1] * y[srcBLen-1]
513 * ....
514 * sum += x[srcALen-2] * y[0] + x[srcALen-1] * y[1]
515 * sum += x[srcALen-1] * y[0]
516 */
517
518 /* In this stage the MAC operations are decreased by 1 for every iteration.
519 The count variable holds the number of MAC operations performed */
520 count = srcBLen - 1U;
521
522 /* Working pointer of inputA */
523 pSrc1 = ((pIn1 + srcALen) - srcBLen) + 1U;
524 px = pSrc1;
525
526 /* Working pointer of inputB */
527 py = pIn2;
528
529 /* -------------------
530 * Stage3 process
531 * ------------------*/
532
533 while (blockSize3 > 0U)
534 {
535 /* Accumulator is made zero for every iteration */
536 sum = 0;
537
538 /* Apply loop unrolling and compute 4 MACs simultaneously. */
539 k = count >> 2U;
540
541 /* First part of the processing with loop unrolling. Compute 4 MACs at a time.
542 ** a second loop below computes MACs for the remaining 1 to 3 samples. */
543 while (k > 0U)
544 {
545 /* Perform the multiply-accumulate */
546 /* sum += x[srcALen - srcBLen + 4] * y[3] */
547 sum = (q31_t) ((((q63_t) sum << 32) +
548 ((q63_t) *px++ * (*py++))) >> 32);
549
550 /* sum += x[srcALen - srcBLen + 3] * y[2] */
551 sum = (q31_t) ((((q63_t) sum << 32) +
552 ((q63_t) *px++ * (*py++))) >> 32);
553
554 /* sum += x[srcALen - srcBLen + 2] * y[1] */
555 sum = (q31_t) ((((q63_t) sum << 32) +
556 ((q63_t) *px++ * (*py++))) >> 32);
557
558 /* sum += x[srcALen - srcBLen + 1] * y[0] */
559 sum = (q31_t) ((((q63_t) sum << 32) +
560 ((q63_t) *px++ * (*py++))) >> 32);
561
562 /* Decrement loop counter */
563 k--;
564 }
565
566 /* If the count is not a multiple of 4, compute any remaining MACs here.
567 ** No loop unrolling is used. */
568 k = count % 0x4U;
569
570 while (k > 0U)
571 {
572 /* Perform the multiply-accumulate */
573 sum = (q31_t) ((((q63_t) sum << 32) +
574 ((q63_t) *px++ * (*py++))) >> 32);
575
576 /* Decrement loop counter */
577 k--;
578 }
579
580 /* Store the result in the accumulator in the destination buffer. */
581 *pOut = sum << 1;
582 /* Destination pointer is updated according to the address modifier, inc */
583 pOut += inc;
584
585 /* Update the inputA and inputB pointers for next MAC calculation */
586 px = ++pSrc1;
587 py = pIn2;
588
589 /* Decrement MAC count */
590 count--;
591
592 /* Decrement loop counter */
593 blockSize3--;
594 }
595
596 }
597
598 /**
599 @} end of Corr group
600 */
601
