1#!/usr/bin/env python3
2
3"""Sanity checks for test data.
4
5This program contains a class for traversing test cases that can be used
6independently of the checks.
7"""
8
9# Copyright The Mbed TLS Contributors
10# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
11
12import argparse
13import glob
14import os
15import re
16import subprocess
17import sys
18
19class ScriptOutputError(ValueError):
20    """A kind of ValueError that indicates we found
21    the script doesn't list test cases in an expected
22    pattern.
23    """
24
25    @property
26    def script_name(self):
27        return super().args[0]
28
29    @property
30    def idx(self):
31        return super().args[1]
32
33    @property
34    def line(self):
35        return super().args[2]
36
37class Results:
38    """Store file and line information about errors or warnings in test suites."""
39
40    def __init__(self, options):
41        self.errors = 0
42        self.warnings = 0
43        self.ignore_warnings = options.quiet
44
45    def error(self, file_name, line_number, fmt, *args):
46        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
47                         format(file_name, line_number, *args))
48        self.errors += 1
49
50    def warning(self, file_name, line_number, fmt, *args):
51        if not self.ignore_warnings:
52            sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
53                             .format(file_name, line_number, *args))
54            self.warnings += 1
55
56class TestDescriptionExplorer:
57    """An iterator over test cases with descriptions.
58
59The test cases that have descriptions are:
60* Individual unit tests (entries in a .data file) in test suites.
61* Individual test cases in ssl-opt.sh.
62
63This is an abstract class. To use it, derive a class that implements
64the process_test_case method, and call walk_all().
65"""
66
67    def process_test_case(self, per_file_state,
68                          file_name, line_number, description):
69        """Process a test case.
70
71per_file_state: an object created by new_per_file_state() at the beginning
72                of each file.
73file_name: a relative path to the file containing the test case.
74line_number: the line number in the given file.
75description: the test case description as a byte string.
76"""
77        raise NotImplementedError
78
79    def new_per_file_state(self):
80        """Return a new per-file state object.
81
82The default per-file state object is None. Child classes that require per-file
83state may override this method.
84"""
85        #pylint: disable=no-self-use
86        return None
87
88    def walk_test_suite(self, data_file_name):
89        """Iterate over the test cases in the given unit test data file."""
90        in_paragraph = False
91        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
92        with open(data_file_name, 'rb') as data_file:
93            for line_number, line in enumerate(data_file, 1):
94                line = line.rstrip(b'\r\n')
95                if not line:
96                    in_paragraph = False
97                    continue
98                if line.startswith(b'#'):
99                    continue
100                if not in_paragraph:
101                    # This is a test case description line.
102                    self.process_test_case(descriptions,
103                                           data_file_name, line_number, line)
104                in_paragraph = True
105
106    def collect_from_script(self, script_name):
107        """Collect the test cases in a script by calling its listing test cases
108option"""
109        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
110        listed = subprocess.check_output(['sh', script_name, '--list-test-cases'])
111        # Assume test file is responsible for printing identical format of
112        # test case description between --list-test-cases and its OUTCOME.CSV
113        #
114        # idx indicates the number of test case since there is no line number
115        # in the script for each test case.
116        for idx, line in enumerate(listed.splitlines()):
117            # We are expecting the script to list the test cases in
118            # `<suite_name>;<description>` pattern.
119            script_outputs = line.split(b';', 1)
120            if len(script_outputs) == 2:
121                suite_name, description = script_outputs
122            else:
123                raise ScriptOutputError(script_name, idx, line.decode("utf-8"))
124
125            self.process_test_case(descriptions,
126                                   suite_name.decode('utf-8'),
127                                   idx,
128                                   description.rstrip())
129
130    @staticmethod
131    def collect_test_directories():
132        """Get the relative path for the TLS and Crypto test directories."""
133        if os.path.isdir('tests'):
134            tests_dir = 'tests'
135        elif os.path.isdir('suites'):
136            tests_dir = '.'
137        elif os.path.isdir('../suites'):
138            tests_dir = '..'
139        directories = [tests_dir]
140        return directories
141
142    def walk_all(self):
143        """Iterate over all named test cases."""
144        test_directories = self.collect_test_directories()
145        for directory in test_directories:
146            for data_file_name in glob.glob(os.path.join(directory, 'suites',
147                                                         '*.data')):
148                self.walk_test_suite(data_file_name)
149
150            for sh_file in ['ssl-opt.sh', 'compat.sh']:
151                sh_file = os.path.join(directory, sh_file)
152                self.collect_from_script(sh_file)
153
154class TestDescriptions(TestDescriptionExplorer):
155    """Collect the available test cases."""
156
157    def __init__(self):
158        super().__init__()
159        self.descriptions = set()
160
161    def process_test_case(self, _per_file_state,
162                          file_name, _line_number, description):
163        """Record an available test case."""
164        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
165        key = ';'.join([base_name, description.decode('utf-8')])
166        self.descriptions.add(key)
167
168def collect_available_test_cases():
169    """Collect the available test cases."""
170    explorer = TestDescriptions()
171    explorer.walk_all()
172    return sorted(explorer.descriptions)
173
174class DescriptionChecker(TestDescriptionExplorer):
175    """Check all test case descriptions.
176
177* Check that each description is valid (length, allowed character set, etc.).
178* Check that there is no duplicated description inside of one test suite.
179"""
180
181    def __init__(self, results):
182        self.results = results
183
184    def new_per_file_state(self):
185        """Dictionary mapping descriptions to their line number."""
186        return {}
187
188    def process_test_case(self, per_file_state,
189                          file_name, line_number, description):
190        """Check test case descriptions for errors."""
191        results = self.results
192        seen = per_file_state
193        if description in seen:
194            results.error(file_name, line_number,
195                          'Duplicate description (also line {})',
196                          seen[description])
197            return
198        if re.search(br'[\t;]', description):
199            results.error(file_name, line_number,
200                          'Forbidden character \'{}\' in description',
201                          re.search(br'[\t;]', description).group(0).decode('ascii'))
202        if re.search(br'[^ -~]', description):
203            results.error(file_name, line_number,
204                          'Non-ASCII character in description')
205        if len(description) > 66:
206            results.warning(file_name, line_number,
207                            'Test description too long ({} > 66)',
208                            len(description))
209        seen[description] = line_number
210
211def main():
212    parser = argparse.ArgumentParser(description=__doc__)
213    parser.add_argument('--list-all',
214                        action='store_true',
215                        help='List all test cases, without doing checks')
216    parser.add_argument('--quiet', '-q',
217                        action='store_true',
218                        help='Hide warnings')
219    parser.add_argument('--verbose', '-v',
220                        action='store_false', dest='quiet',
221                        help='Show warnings (default: on; undoes --quiet)')
222    options = parser.parse_args()
223    if options.list_all:
224        descriptions = collect_available_test_cases()
225        sys.stdout.write('\n'.join(descriptions + ['']))
226        return
227    results = Results(options)
228    checker = DescriptionChecker(results)
229    try:
230        checker.walk_all()
231    except ScriptOutputError as e:
232        results.error(e.script_name, e.idx,
233                      '"{}" should be listed as "<suite_name>;<description>"',
234                      e.line)
235    if (results.warnings or results.errors) and not options.quiet:
236        sys.stderr.write('{}: {} errors, {} warnings\n'
237                         .format(sys.argv[0], results.errors, results.warnings))
238    sys.exit(1 if results.errors else 0)
239
240if __name__ == '__main__':
241    main()
242