1#!/usr/bin/env python3
2
3"""Sanity checks for test data.
4
5This program contains a class for traversing test cases that can be used
6independently of the checks.
7"""
8
9# Copyright The Mbed TLS Contributors
10# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
11
12import argparse
13import glob
14import os
15import re
16import subprocess
17import sys
18
19class Results:
20    """Store file and line information about errors or warnings in test suites."""
21
22    def __init__(self, options):
23        self.errors = 0
24        self.warnings = 0
25        self.ignore_warnings = options.quiet
26
27    def error(self, file_name, line_number, fmt, *args):
28        sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n').
29                         format(file_name, line_number, *args))
30        self.errors += 1
31
32    def warning(self, file_name, line_number, fmt, *args):
33        if not self.ignore_warnings:
34            sys.stderr.write(('{}:{}:Warning:' + fmt + '\n')
35                             .format(file_name, line_number, *args))
36            self.warnings += 1
37
38class TestDescriptionExplorer:
39    """An iterator over test cases with descriptions.
40
41The test cases that have descriptions are:
42* Individual unit tests (entries in a .data file) in test suites.
43* Individual test cases in ssl-opt.sh.
44
45This is an abstract class. To use it, derive a class that implements
46the process_test_case method, and call walk_all().
47"""
48
49    def process_test_case(self, per_file_state,
50                          file_name, line_number, description):
51        """Process a test case.
52
53per_file_state: an object created by new_per_file_state() at the beginning
54                of each file.
55file_name: a relative path to the file containing the test case.
56line_number: the line number in the given file.
57description: the test case description as a byte string.
58"""
59        raise NotImplementedError
60
61    def new_per_file_state(self):
62        """Return a new per-file state object.
63
64The default per-file state object is None. Child classes that require per-file
65state may override this method.
66"""
67        #pylint: disable=no-self-use
68        return None
69
70    def walk_test_suite(self, data_file_name):
71        """Iterate over the test cases in the given unit test data file."""
72        in_paragraph = False
73        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
74        with open(data_file_name, 'rb') as data_file:
75            for line_number, line in enumerate(data_file, 1):
76                line = line.rstrip(b'\r\n')
77                if not line:
78                    in_paragraph = False
79                    continue
80                if line.startswith(b'#'):
81                    continue
82                if not in_paragraph:
83                    # This is a test case description line.
84                    self.process_test_case(descriptions,
85                                           data_file_name, line_number, line)
86                in_paragraph = True
87
88    def walk_ssl_opt_sh(self, file_name):
89        """Iterate over the test cases in ssl-opt.sh or a file with a similar format."""
90        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
91        with open(file_name, 'rb') as file_contents:
92            for line_number, line in enumerate(file_contents, 1):
93                # Assume that all run_test calls have the same simple form
94                # with the test description entirely on the same line as the
95                # function name.
96                m = re.match(br'\s*run_test\s+"((?:[^\\"]|\\.)*)"', line)
97                if not m:
98                    continue
99                description = m.group(1)
100                self.process_test_case(descriptions,
101                                       file_name, line_number, description)
102
103    def walk_compat_sh(self, file_name):
104        """Iterate over the test cases compat.sh with a similar format."""
105        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
106        compat_cmd = ['sh', file_name, '--list-test-case']
107        compat_output = subprocess.check_output(compat_cmd)
108        # Assume compat.sh is responsible for printing identical format of
109        # test case description between --list-test-case and its OUTCOME.CSV
110        description = compat_output.strip().split(b'\n')
111        # idx indicates the number of test case since there is no line number
112        # in `compat.sh` for each test case.
113        for idx, descrip in enumerate(description):
114            self.process_test_case(descriptions, file_name, idx, descrip)
115
116    @staticmethod
117    def collect_test_directories():
118        """Get the relative path for the TLS and Crypto test directories."""
119        if os.path.isdir('tests'):
120            tests_dir = 'tests'
121        elif os.path.isdir('suites'):
122            tests_dir = '.'
123        elif os.path.isdir('../suites'):
124            tests_dir = '..'
125        directories = [tests_dir]
126        return directories
127
128    def walk_all(self):
129        """Iterate over all named test cases."""
130        test_directories = self.collect_test_directories()
131        for directory in test_directories:
132            for data_file_name in glob.glob(os.path.join(directory, 'suites',
133                                                         '*.data')):
134                self.walk_test_suite(data_file_name)
135            ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
136            if os.path.exists(ssl_opt_sh):
137                self.walk_ssl_opt_sh(ssl_opt_sh)
138            for ssl_opt_file_name in glob.glob(os.path.join(directory, 'opt-testcases',
139                                                            '*.sh')):
140                self.walk_ssl_opt_sh(ssl_opt_file_name)
141            compat_sh = os.path.join(directory, 'compat.sh')
142            if os.path.exists(compat_sh):
143                self.walk_compat_sh(compat_sh)
144
145class TestDescriptions(TestDescriptionExplorer):
146    """Collect the available test cases."""
147
148    def __init__(self):
149        super().__init__()
150        self.descriptions = set()
151
152    def process_test_case(self, _per_file_state,
153                          file_name, _line_number, description):
154        """Record an available test case."""
155        base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name))
156        key = ';'.join([base_name, description.decode('utf-8')])
157        self.descriptions.add(key)
158
159def collect_available_test_cases():
160    """Collect the available test cases."""
161    explorer = TestDescriptions()
162    explorer.walk_all()
163    return sorted(explorer.descriptions)
164
165class DescriptionChecker(TestDescriptionExplorer):
166    """Check all test case descriptions.
167
168* Check that each description is valid (length, allowed character set, etc.).
169* Check that there is no duplicated description inside of one test suite.
170"""
171
172    def __init__(self, results):
173        self.results = results
174
175    def new_per_file_state(self):
176        """Dictionary mapping descriptions to their line number."""
177        return {}
178
179    def process_test_case(self, per_file_state,
180                          file_name, line_number, description):
181        """Check test case descriptions for errors."""
182        results = self.results
183        seen = per_file_state
184        if description in seen:
185            results.error(file_name, line_number,
186                          'Duplicate description (also line {})',
187                          seen[description])
188            return
189        if re.search(br'[\t;]', description):
190            results.error(file_name, line_number,
191                          'Forbidden character \'{}\' in description',
192                          re.search(br'[\t;]', description).group(0).decode('ascii'))
193        if re.search(br'[^ -~]', description):
194            results.error(file_name, line_number,
195                          'Non-ASCII character in description')
196        if len(description) > 66:
197            results.warning(file_name, line_number,
198                            'Test description too long ({} > 66)',
199                            len(description))
200        seen[description] = line_number
201
202def main():
203    parser = argparse.ArgumentParser(description=__doc__)
204    parser.add_argument('--list-all',
205                        action='store_true',
206                        help='List all test cases, without doing checks')
207    parser.add_argument('--quiet', '-q',
208                        action='store_true',
209                        help='Hide warnings')
210    parser.add_argument('--verbose', '-v',
211                        action='store_false', dest='quiet',
212                        help='Show warnings (default: on; undoes --quiet)')
213    options = parser.parse_args()
214    if options.list_all:
215        descriptions = collect_available_test_cases()
216        sys.stdout.write('\n'.join(descriptions + ['']))
217        return
218    results = Results(options)
219    checker = DescriptionChecker(results)
220    checker.walk_all()
221    if (results.warnings or results.errors) and not options.quiet:
222        sys.stderr.write('{}: {} errors, {} warnings\n'
223                         .format(sys.argv[0], results.errors, results.warnings))
224    sys.exit(1 if results.errors else 0)
225
226if __name__ == '__main__':
227    main()
228