1#!/usr/bin/env python3
2#
3# Minimal failing test sequence finder
4# Copyright (c) 2022, Qualcomm Innovation Center, Inc.
5#
6# This software may be distributed under the terms of the BSD license.
7# See README for more details.
8
9import subprocess
10import sys
11from colorama import Fore, Style
12
13def red(s, bright=False):
14    tmp = Style.BRIGHT if bright else ''
15    return tmp + Fore.RED + s + Style.RESET_ALL
16
17def yellow(s, bright=False):
18    tmp = Style.BRIGHT if bright else ''
19    return tmp + Fore.YELLOW + s + Style.RESET_ALL
20
21def bright(s):
22    return Style.BRIGHT + s + Style.RESET_ALL
23
24def run_tests(tests):
25    print(yellow("Run test sequence: ") + ' '.join(tests))
26    arg = ['./vm-run.sh'] + tests
27    cmd = subprocess.Popen(arg, stdout=subprocess.PIPE)
28    out = cmd.stdout.read().decode()
29    found = False
30    for i in out.splitlines():
31        if i.startswith('FAIL '):
32            t = i.split(' ')[1]
33            if t == tests[-1]:
34                found = True
35            else:
36                print(red("Unexpected FAIL: ", bright=True) + t)
37                return None
38    return found
39
40def reduce(tests):
41    if len(tests) < 2:
42        return None
43
44    # Try to remove first half of the test cases to speed up the initial process
45    if len(tests) > 10:
46        a = list(tests[int(len(tests) / 2):])
47        res = run_tests(a)
48        if res is None:
49            return None
50        if res:
51            return a
52
53    # Try to remove test cases one-by-one (starting with larger groups to speed
54    # up)
55    for count in [27, 9, 6, 3, 1]:
56        for i in range(0, len(tests) - count, count):
57            b = list(tests)
58            del b[i:i + count]
59            if len(b) < 2:
60                continue
61            res = run_tests(b)
62            if res is None:
63                return None
64            if res:
65                return b
66
67    return None
68
69def main():
70    tests = sys.argv[1:]
71    num_tests = len(tests)
72    if not run_tests(tests):
73        print(red("Full test sequence did not result in an error", bright=True))
74        return
75    while True:
76        new_tests = reduce(tests)
77        if (not new_tests) or len(new_tests) == len(tests):
78            break
79        tests = new_tests
80        print(yellow("Found a shorter sequence: ", bright=True) + ' '.join(tests))
81    if len(tests) < num_tests:
82        print(bright("Minimal sequence:"))
83        print(' '.join(tests))
84    else:
85        print(yellow("Could not remove any test cases without losing the failure"))
86
87if __name__ == "__main__":
88    main()
89