1 //! Parallel testing.
2 //!
3 //! mcuboot simulator is strictly single threaded, as there is a lock around running the C startup
4 //! code, because it contains numerous global variables.
5 //!
6 //! To help speed up testing, the Workflow configuration defines all of the configurations that can
7 //! be run in parallel. Fortunately, cargo works well this way, and these can be run by simply
8 //! using subprocess for each particular thread.
9 //!
10 //! For now, we assume all of the features are listed under
11 //! jobs->environment->strategy->matric->features
12
13 use chrono::Local;
14 use log::{debug, error, warn};
15 use std::{
16 collections::HashSet,
17 env,
18 fs::{self, OpenOptions},
19 io::{ErrorKind, stdout, Write},
20 process::Command,
21 result,
22 sync::{
23 Arc,
24 Mutex,
25 },
26 thread,
27 time::Duration,
28 };
29 use std_semaphore::Semaphore;
30 use yaml_rust::{
31 Yaml,
32 YamlLoader,
33 };
34
35 type Result<T> = result::Result<T, failure::Error>;
36
main() -> Result<()>37 fn main() -> Result<()> {
38 env_logger::init();
39
40 let workflow_text = fs::read_to_string("../.github/workflows/sim.yaml")?;
41 let workflow = YamlLoader::load_from_str(&workflow_text)?;
42
43 let ncpus = num_cpus::get();
44 let limiter = Arc::new(Semaphore::new(ncpus as isize));
45
46 let matrix = Matrix::from_yaml(&workflow);
47
48 let mut children = vec![];
49 let state = State::new(matrix.envs.len());
50 let st2 = state.clone();
51 let _status = thread::spawn(move || {
52 loop {
53 thread::sleep(Duration::new(15, 0));
54 st2.lock().unwrap().status();
55 }
56 });
57 for env in matrix.envs {
58 let state = state.clone();
59 let limiter = limiter.clone();
60
61 let child = thread::spawn(move || {
62 let _run = limiter.access();
63 state.lock().unwrap().start(&env);
64 let out = env.run();
65 state.lock().unwrap().done(&env, out);
66 });
67 children.push(child);
68 }
69
70 for child in children {
71 child.join().unwrap();
72 }
73
74 println!();
75
76 Ok(())
77 }
78
79 /// State, for printing status.
80 struct State {
81 running: HashSet<String>,
82 done: HashSet<String>,
83 total: usize,
84 }
85
86 /// Result of a test run.
87 struct TestResult {
88 /// Was this run successful.
89 success: bool,
90
91 /// The captured output.
92 output: Vec<u8>,
93 }
94
95 impl State {
new(total: usize) -> Arc<Mutex<State>>96 fn new(total: usize) -> Arc<Mutex<State>> {
97 Arc::new(Mutex::new(State {
98 running: HashSet::new(),
99 done: HashSet::new(),
100 total,
101 }))
102 }
103
start(&mut self, fs: &FeatureSet)104 fn start(&mut self, fs: &FeatureSet) {
105 let key = fs.textual();
106 if self.running.contains(&key) || self.done.contains(&key) {
107 warn!("Duplicate: {:?}", key);
108 }
109 debug!("Starting: {} ({} running)", key, self.running.len() + 1);
110 self.running.insert(key);
111 self.status();
112 }
113
done(&mut self, fs: &FeatureSet, output: Result<TestResult>)114 fn done(&mut self, fs: &FeatureSet, output: Result<TestResult>) {
115 let key = fs.textual();
116 self.running.remove(&key);
117 self.done.insert(key.clone());
118 match output {
119 Ok(output) => {
120 if !output.success || log_all() {
121 // Write the output into a file.
122 let mut count = 1;
123 let (mut fd, logname) = loop {
124 let base = if output.success { "success" } else { "failure" };
125 let name = format!("./{}-{:04}.log", base, count);
126 count += 1;
127 match OpenOptions::new()
128 .create_new(true)
129 .write(true)
130 .open(&name)
131 {
132 Ok(file) => break (file, name),
133 Err(ref err) if err.kind() == ErrorKind::AlreadyExists => continue,
134 Err(err) => {
135 error!("Unable to write log file to current directory: {:?}", err);
136 return;
137 }
138 }
139 };
140 fd.write_all(&output.output).unwrap();
141 if !output.success {
142 error!("Failure {} log:{:?} ({} running)", key, logname,
143 self.running.len());
144 }
145 }
146 }
147 Err(err) => {
148 error!("Unable to run test {:?} ({:?}", key, err);
149 }
150 }
151 self.status();
152 }
153
status(&self)154 fn status(&self) {
155 let running = self.running.len();
156 let done = self.done.len();
157 print!(" {} running ({}/{}/{} done)\r", running, done, running + done, self.total);
158 stdout().flush().unwrap();
159 }
160 }
161
162 /// The extracted configurations from the workflow config
163 #[derive(Debug)]
164 struct Matrix {
165 envs: Vec<FeatureSet>,
166 }
167
168 #[derive(Debug, Eq, Hash, PartialEq)]
169 struct FeatureSet {
170 // The environment variable to set.
171 env: String,
172 // The successive values to set it to.
173 values: Vec<String>,
174 }
175
176 impl Matrix {
from_yaml(yaml: &[Yaml]) -> Matrix177 fn from_yaml(yaml: &[Yaml]) -> Matrix {
178 let mut envs = vec![];
179
180 let mut all_tests = HashSet::new();
181
182 for y in yaml {
183 let m = match lookup_matrix(y) {
184 Some (m) => m,
185 None => continue,
186 };
187 for elt in m {
188 let elt = match elt.as_str() {
189 None => {
190 warn!("Unexpected yaml: {:?}", elt);
191 continue;
192 }
193 Some(e) => e,
194 };
195 let fset = FeatureSet::decode(elt);
196
197 if false {
198 // Respect the groupings in the `.workflow.yml` file.
199 envs.push(fset);
200 } else {
201 // Break each test up so we can run more in
202 // parallel.
203 let env = fset.env.clone();
204 for val in fset.values {
205 if !all_tests.contains(&val) {
206 all_tests.insert(val.clone());
207 envs.push(FeatureSet {
208 env: env.clone(),
209 values: vec![val],
210 });
211 } else {
212 warn!("Duplicate: {:?}: {:?}", env, val);
213 }
214 }
215 }
216 }
217 }
218
219 Matrix {
220 envs,
221 }
222 }
223 }
224
225 impl FeatureSet {
decode(text: &str) -> FeatureSet226 fn decode(text: &str) -> FeatureSet {
227 // The github workflow is just a space separated set of values.
228 let values: Vec<_> = text
229 .split(',')
230 .map(|s| s.to_string())
231 .collect();
232 FeatureSet {
233 env: "MULTI_FEATURES".to_string(),
234 values,
235 }
236 }
237
238 /// Run a test for this given feature set. Output is captured and will be returned if there is
239 /// an error. Each will be run successively, and the first failure will be returned.
240 /// Otherwise, it returns None, which means everything worked.
run(&self) -> Result<TestResult>241 fn run(&self) -> Result<TestResult> {
242 let mut output = vec![];
243 let mut success = true;
244 for v in &self.values {
245 let cmdout = Command::new("bash")
246 .arg("./ci/sim_run.sh")
247 .current_dir("..")
248 .env(&self.env, v)
249 .output()?;
250 // Grab the output for logging, etc.
251 writeln!(&mut output, "Test {} {}",
252 if cmdout.status.success() { "success" } else { "FAILURE" },
253 self.textual())?;
254 writeln!(&mut output, "time: {}", Local::now().to_rfc3339())?;
255 writeln!(&mut output, "----------------------------------------")?;
256 writeln!(&mut output, "stdout:")?;
257 output.extend(&cmdout.stdout);
258 writeln!(&mut output, "----------------------------------------")?;
259 writeln!(&mut output, "stderr:")?;
260 output.extend(&cmdout.stderr);
261 if !cmdout.status.success() {
262 success = false;
263 }
264 }
265 Ok(TestResult { success, output })
266 }
267
268 /// Convert this feature set into a textual representation
textual(&self) -> String269 fn textual(&self) -> String {
270 use std::fmt::Write;
271
272 let mut buf = String::new();
273
274 write!(&mut buf, "{}:", self.env).unwrap();
275 for v in &self.values {
276 write!(&mut buf, " {}", v).unwrap();
277 }
278
279 buf
280 }
281 }
282
lookup_matrix(y: &Yaml) -> Option<&Vec<Yaml>>283 fn lookup_matrix(y: &Yaml) -> Option<&Vec<Yaml>> {
284 let jobs = Yaml::String("jobs".to_string());
285 let environment = Yaml::String("environment".to_string());
286 let strategy = Yaml::String("strategy".to_string());
287 let matrix = Yaml::String("matrix".to_string());
288 let features = Yaml::String("features".to_string());
289 y
290 .as_hash()?.get(&jobs)?
291 .as_hash()?.get(&environment)?
292 .as_hash()?.get(&strategy)?
293 .as_hash()?.get(&matrix)?
294 .as_hash()?.get(&features)?
295 .as_vec()
296 }
297
298 /// Query if we should be logging all tests and not only failures.
log_all() -> bool299 fn log_all() -> bool {
300 env::var("PTEST_LOG_ALL").is_ok()
301 }
302
