1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6
7 http://www.apache.org/licenses/LICENSE-2.0
8
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15
16 #include "tensorflow/lite/micro/examples/micro_speech/recognize_commands.h"
17
18 #include <limits>
19
RecognizeCommands(tflite::ErrorReporter * error_reporter,int32_t average_window_duration_ms,uint8_t detection_threshold,int32_t suppression_ms,int32_t minimum_count)20 RecognizeCommands::RecognizeCommands(tflite::ErrorReporter* error_reporter,
21 int32_t average_window_duration_ms,
22 uint8_t detection_threshold,
23 int32_t suppression_ms,
24 int32_t minimum_count)
25 : error_reporter_(error_reporter),
26 average_window_duration_ms_(average_window_duration_ms),
27 detection_threshold_(detection_threshold),
28 suppression_ms_(suppression_ms),
29 minimum_count_(minimum_count),
30 previous_results_(error_reporter) {
31 previous_top_label_ = "silence";
32 previous_top_label_time_ = std::numeric_limits<int32_t>::min();
33 }
34
ProcessLatestResults(const TfLiteTensor * latest_results,const int32_t current_time_ms,const char ** found_command,uint8_t * score,bool * is_new_command)35 TfLiteStatus RecognizeCommands::ProcessLatestResults(
36 const TfLiteTensor* latest_results, const int32_t current_time_ms,
37 const char** found_command, uint8_t* score, bool* is_new_command) {
38 if ((latest_results->dims->size != 2) ||
39 (latest_results->dims->data[0] != 1) ||
40 (latest_results->dims->data[1] != kCategoryCount)) {
41 TF_LITE_REPORT_ERROR(
42 error_reporter_,
43 "The results for recognition should contain %d elements, but there are "
44 "%d in an %d-dimensional shape",
45 kCategoryCount, latest_results->dims->data[1],
46 latest_results->dims->size);
47 return kTfLiteError;
48 }
49
50 if (latest_results->type != kTfLiteInt8) {
51 TF_LITE_REPORT_ERROR(
52 error_reporter_,
53 "The results for recognition should be int8_t elements, but are %d",
54 latest_results->type);
55 return kTfLiteError;
56 }
57
58 if ((!previous_results_.empty()) &&
59 (current_time_ms < previous_results_.front().time_)) {
60 TF_LITE_REPORT_ERROR(
61 error_reporter_,
62 "Results must be fed in increasing time order, but received a "
63 "timestamp of %d that was earlier than the previous one of %d",
64 current_time_ms, previous_results_.front().time_);
65 return kTfLiteError;
66 }
67
68 // Add the latest results to the head of the queue.
69 previous_results_.push_back({current_time_ms, latest_results->data.int8});
70
71 // Prune any earlier results that are too old for the averaging window.
72 const int64_t time_limit = current_time_ms - average_window_duration_ms_;
73 while ((!previous_results_.empty()) &&
74 previous_results_.front().time_ < time_limit) {
75 previous_results_.pop_front();
76 }
77
78 // If there are too few results, assume the result will be unreliable and
79 // bail.
80 const int64_t how_many_results = previous_results_.size();
81 const int64_t earliest_time = previous_results_.front().time_;
82 const int64_t samples_duration = current_time_ms - earliest_time;
83 if ((how_many_results < minimum_count_) ||
84 (samples_duration < (average_window_duration_ms_ / 4))) {
85 *found_command = previous_top_label_;
86 *score = 0;
87 *is_new_command = false;
88 return kTfLiteOk;
89 }
90
91 // Calculate the average score across all the results in the window.
92 int32_t average_scores[kCategoryCount];
93 for (int offset = 0; offset < previous_results_.size(); ++offset) {
94 PreviousResultsQueue::Result previous_result =
95 previous_results_.from_front(offset);
96 const int8_t* scores = previous_result.scores;
97 for (int i = 0; i < kCategoryCount; ++i) {
98 if (offset == 0) {
99 average_scores[i] = scores[i] + 128;
100 } else {
101 average_scores[i] += scores[i] + 128;
102 }
103 }
104 }
105 for (int i = 0; i < kCategoryCount; ++i) {
106 average_scores[i] /= how_many_results;
107 }
108
109 // Find the current highest scoring category.
110 int current_top_index = 0;
111 int32_t current_top_score = 0;
112 for (int i = 0; i < kCategoryCount; ++i) {
113 if (average_scores[i] > current_top_score) {
114 current_top_score = average_scores[i];
115 current_top_index = i;
116 }
117 }
118 const char* current_top_label = kCategoryLabels[current_top_index];
119
120 // If we've recently had another label trigger, assume one that occurs too
121 // soon afterwards is a bad result.
122 int64_t time_since_last_top;
123 if ((previous_top_label_ == kCategoryLabels[0]) ||
124 (previous_top_label_time_ == std::numeric_limits<int32_t>::min())) {
125 time_since_last_top = std::numeric_limits<int32_t>::max();
126 } else {
127 time_since_last_top = current_time_ms - previous_top_label_time_;
128 }
129 if ((current_top_score > detection_threshold_) &&
130 ((current_top_label != previous_top_label_) ||
131 (time_since_last_top > suppression_ms_))) {
132 previous_top_label_ = current_top_label;
133 previous_top_label_time_ = current_time_ms;
134 *is_new_command = true;
135 } else {
136 *is_new_command = false;
137 }
138 *found_command = current_top_label;
139 *score = current_top_score;
140
141 return kTfLiteOk;
142 }
143
