1 /* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #include "tensorflow/lite/micro/examples/micro_speech/audio_provider.h"
17 
18 #include "AUDIO_DISCO_F746NG.h"
19 #include "SDRAM_DISCO_F746NG.h"
20 #include "mbed.h"  // NOLINT
21 #include "tensorflow/lite/micro/examples/micro_speech/micro_features/micro_model_settings.h"
22 
23 namespace {
24 
25 bool g_is_audio_initialized = false;
26 constexpr int kAudioCaptureBufferSize = kAudioSampleFrequency * 0.5;
27 int16_t g_audio_capture_buffer[kAudioCaptureBufferSize];
28 int16_t g_audio_output_buffer[kMaxAudioSampleSize];
29 int32_t g_latest_audio_timestamp = 0;
30 
31 // For a full example of how to access audio on the STM32F746NG board, see
32 // https://os.mbed.com/teams/ST/code/DISCO-F746NG_AUDIO_demo/
33 AUDIO_DISCO_F746NG g_audio_device;
34 SDRAM_DISCO_F746NG g_sdram_device;
35 
36 typedef enum {
37   BUFFER_OFFSET_NONE = 0,
38   BUFFER_OFFSET_HALF = 1,
39   BUFFER_OFFSET_FULL = 2,
40 } BUFFER_StateTypeDef;
41 
42 #define AUDIO_BLOCK_SIZE ((uint32_t)2048)
43 #define AUDIO_BUFFER_IN SDRAM_DEVICE_ADDR /* In SDRAM */
44 #define AUDIO_BUFFER_OUT \
45   (SDRAM_DEVICE_ADDR + (AUDIO_BLOCK_SIZE * 2)) /* In SDRAM */
46 __IO uint32_t g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
47 
SetSysClock_PLL_HSE_200MHz()48 uint8_t SetSysClock_PLL_HSE_200MHz() {
49   RCC_ClkInitTypeDef RCC_ClkInitStruct;
50   RCC_OscInitTypeDef RCC_OscInitStruct;
51 
52   // Enable power clock
53   __PWR_CLK_ENABLE();
54 
55   // Enable HSE oscillator and activate PLL with HSE as source
56   RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSE;
57   RCC_OscInitStruct.HSEState = RCC_HSE_ON; /* External xtal on OSC_IN/OSC_OUT */
58 
59   // Warning: this configuration is for a 25 MHz xtal clock only
60   RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON;
61   RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSE;
62   RCC_OscInitStruct.PLL.PLLM = 25;   // VCO input clock = 1 MHz (25 MHz / 25)
63   RCC_OscInitStruct.PLL.PLLN = 400;  // VCO output clock = 400 MHz (1 MHz * 400)
64   RCC_OscInitStruct.PLL.PLLP = RCC_PLLP_DIV2;  // PLLCLK = 200 MHz (400 MHz / 2)
65   RCC_OscInitStruct.PLL.PLLQ = 8;  // USB clock = 50 MHz (400 MHz / 8)
66 
67   if (HAL_RCC_OscConfig(&RCC_OscInitStruct) != HAL_OK) {
68     return 0;  // FAIL
69   }
70 
71   // Activate the OverDrive to reach the 216 MHz Frequency
72   if (HAL_PWREx_EnableOverDrive() != HAL_OK) {
73     return 0;  // FAIL
74   }
75 
76   // Select PLL as system clock source and configure the HCLK, PCLK1 and PCLK2
77   // clocks dividers
78   RCC_ClkInitStruct.ClockType = (RCC_CLOCKTYPE_SYSCLK | RCC_CLOCKTYPE_HCLK |
79                                  RCC_CLOCKTYPE_PCLK1 | RCC_CLOCKTYPE_PCLK2);
80   RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_PLLCLK;  // 200 MHz
81   RCC_ClkInitStruct.AHBCLKDivider = RCC_SYSCLK_DIV1;         // 200 MHz
82   RCC_ClkInitStruct.APB1CLKDivider = RCC_HCLK_DIV4;          //  50 MHz
83   RCC_ClkInitStruct.APB2CLKDivider = RCC_HCLK_DIV2;          // 100 MHz
84 
85   if (HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_7) != HAL_OK) {
86     return 0;  // FAIL
87   }
88   HAL_RCC_MCOConfig(RCC_MCO1, RCC_MCO1SOURCE_HSE, RCC_MCODIV_4);
89   return 1;  // OK
90 }
91 
InitAudioRecording(tflite::ErrorReporter * error_reporter)92 TfLiteStatus InitAudioRecording(tflite::ErrorReporter* error_reporter) {
93   SetSysClock_PLL_HSE_200MHz();
94 
95   // Initialize SDRAM buffers.
96   memset((uint16_t*)AUDIO_BUFFER_IN, 0, AUDIO_BLOCK_SIZE * 2);
97   memset((uint16_t*)AUDIO_BUFFER_OUT, 0, AUDIO_BLOCK_SIZE * 2);
98   g_audio_rec_buffer_state = BUFFER_OFFSET_NONE;
99 
100   // Start Recording.
101   g_audio_device.IN_Record((uint16_t*)AUDIO_BUFFER_IN, AUDIO_BLOCK_SIZE);
102 
103   // Also play results out to headphone jack.
104   g_audio_device.OUT_SetAudioFrameSlot(CODEC_AUDIOFRAME_SLOT_02);
105   g_audio_device.OUT_Play((uint16_t*)AUDIO_BUFFER_OUT, AUDIO_BLOCK_SIZE * 2);
106 
107   return kTfLiteOk;
108 }
109 
CaptureSamples(const int16_t * sample_data)110 void CaptureSamples(const int16_t* sample_data) {
111   const int sample_size = AUDIO_BLOCK_SIZE / (sizeof(int16_t) * 2);
112   const int32_t time_in_ms =
113       g_latest_audio_timestamp + (sample_size / (kAudioSampleFrequency / 1000));
114 
115   const int32_t start_sample_offset =
116       g_latest_audio_timestamp * (kAudioSampleFrequency / 1000);
117   for (int i = 0; i < sample_size; ++i) {
118     const int capture_index =
119         (start_sample_offset + i) % kAudioCaptureBufferSize;
120     g_audio_capture_buffer[capture_index] =
121         (sample_data[(i * 2) + 0] / 2) + (sample_data[(i * 2) + 1] / 2);
122   }
123   // This is how we let the outside world know that new audio data has arrived.
124   g_latest_audio_timestamp = time_in_ms;
125 }
126 
127 }  // namespace
128 
129 // These callbacks need to be linkable symbols, because they override weak
130 // default versions.
BSP_AUDIO_IN_TransferComplete_CallBack(void)131 void BSP_AUDIO_IN_TransferComplete_CallBack(void) {
132   g_audio_rec_buffer_state = BUFFER_OFFSET_FULL;
133   /* Copy recorded 1st half block */
134   memcpy((uint16_t*)(AUDIO_BUFFER_OUT), (uint16_t*)(AUDIO_BUFFER_IN),
135          AUDIO_BLOCK_SIZE);
136   CaptureSamples(reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN));
137   return;
138 }
139 
140 // Another weak symbol override.
BSP_AUDIO_IN_HalfTransfer_CallBack(void)141 void BSP_AUDIO_IN_HalfTransfer_CallBack(void) {
142   g_audio_rec_buffer_state = BUFFER_OFFSET_HALF;
143   /* Copy recorded 2nd half block */
144   memcpy((uint16_t*)(AUDIO_BUFFER_OUT + (AUDIO_BLOCK_SIZE)),
145          (uint16_t*)(AUDIO_BUFFER_IN + (AUDIO_BLOCK_SIZE)), AUDIO_BLOCK_SIZE);
146   CaptureSamples(
147       reinterpret_cast<int16_t*>(AUDIO_BUFFER_IN + AUDIO_BLOCK_SIZE));
148   return;
149 }
150 
151 // Main entry point for getting audio data.
GetAudioSamples(tflite::ErrorReporter * error_reporter,int start_ms,int duration_ms,int * audio_samples_size,int16_t ** audio_samples)152 TfLiteStatus GetAudioSamples(tflite::ErrorReporter* error_reporter,
153                              int start_ms, int duration_ms,
154                              int* audio_samples_size, int16_t** audio_samples) {
155   if (!g_is_audio_initialized) {
156     TfLiteStatus init_status = InitAudioRecording(error_reporter);
157     if (init_status != kTfLiteOk) {
158       return init_status;
159     }
160     g_is_audio_initialized = true;
161   }
162   // This should only be called when the main thread notices that the latest
163   // audio sample data timestamp has changed, so that there's new data in the
164   // capture ring buffer. The ring buffer will eventually wrap around and
165   // overwrite the data, but the assumption is that the main thread is checking
166   // often enough and the buffer is large enough that this call will be made
167   // before that happens.
168   const int start_offset = start_ms * (kAudioSampleFrequency / 1000);
169   const int duration_sample_count =
170       duration_ms * (kAudioSampleFrequency / 1000);
171   for (int i = 0; i < duration_sample_count; ++i) {
172     const int capture_index = (start_offset + i) % kAudioCaptureBufferSize;
173     g_audio_output_buffer[i] = g_audio_capture_buffer[capture_index];
174   }
175 
176   *audio_samples_size = kMaxAudioSampleSize;
177   *audio_samples = g_audio_output_buffer;
178   return kTfLiteOk;
179 }
180 
LatestAudioTimestamp()181 int32_t LatestAudioTimestamp() { return g_latest_audio_timestamp; }
182