1"""Common code for test data generation.
2
3This module defines classes that are of general use to automatically
4generate .data files for unit tests, as well as a main function.
5
6These are used both by generate_psa_tests.py and generate_bignum_tests.py.
7"""
8
9# Copyright The Mbed TLS Contributors
10# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
11#
12
13import argparse
14import os
15import posixpath
16import re
17import inspect
18
19from abc import ABCMeta, abstractmethod
20from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
21
22from . import build_tree
23from . import test_case
24
25T = TypeVar('T') #pylint: disable=invalid-name
26
27
28class BaseTest(metaclass=ABCMeta):
29    """Base class for test case generation.
30
31    Attributes:
32        count: Counter for test cases from this class.
33        case_description: Short description of the test case. This may be
34            automatically generated using the class, or manually set.
35        dependencies: A list of dependencies required for the test case.
36        show_test_count: Toggle for inclusion of `count` in the test description.
37        test_function: Test function which the class generates cases for.
38        test_name: A common name or description of the test function. This can
39            be `test_function`, a clearer equivalent, or a short summary of the
40            test function's purpose.
41    """
42    count = 0
43    case_description = ""
44    dependencies = [] # type: List[str]
45    show_test_count = True
46    test_function = ""
47    test_name = ""
48
49    def __new__(cls, *args, **kwargs):
50        # pylint: disable=unused-argument
51        cls.count += 1
52        return super().__new__(cls)
53
54    @abstractmethod
55    def arguments(self) -> List[str]:
56        """Get the list of arguments for the test case.
57
58        Override this method to provide the list of arguments required for
59        the `test_function`.
60
61        Returns:
62            List of arguments required for the test function.
63        """
64        raise NotImplementedError
65
66    def description(self) -> str:
67        """Create a test case description.
68
69        Creates a description of the test case, including a name for the test
70        function, an optional case count, and a description of the specific
71        test case. This should inform a reader what is being tested, and
72        provide context for the test case.
73
74        Returns:
75            Description for the test case.
76        """
77        if self.show_test_count:
78            return "{} #{} {}".format(
79                self.test_name, self.count, self.case_description
80                ).strip()
81        else:
82            return "{} {}".format(self.test_name, self.case_description).strip()
83
84
85    def create_test_case(self) -> test_case.TestCase:
86        """Generate TestCase from the instance."""
87        tc = test_case.TestCase()
88        tc.set_description(self.description())
89        tc.set_function(self.test_function)
90        tc.set_arguments(self.arguments())
91        tc.set_dependencies(self.dependencies)
92
93        return tc
94
95    @classmethod
96    @abstractmethod
97    def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
98        """Generate test cases for the class test function.
99
100        This will be called in classes where `test_function` is set.
101        Implementations should yield TestCase objects, by creating instances
102        of the class with appropriate input data, and then calling
103        `create_test_case()` on each.
104        """
105        raise NotImplementedError
106
107
108class BaseTarget:
109    #pylint: disable=too-few-public-methods
110    """Base target for test case generation.
111
112    Child classes of this class represent an output file, and can be referred
113    to as file targets. These indicate where test cases will be written to for
114    all subclasses of the file target, which is set by `target_basename`.
115
116    Attributes:
117        target_basename: Basename of file to write generated tests to. This
118            should be specified in a child class of BaseTarget.
119    """
120    target_basename = ""
121
122    @classmethod
123    def generate_tests(cls) -> Iterator[test_case.TestCase]:
124        """Generate test cases for the class and its subclasses.
125
126        In classes with `test_function` set, `generate_function_tests()` is
127        called to generate test cases first.
128
129        In all classes, this method will iterate over its subclasses, and
130        yield from `generate_tests()` in each. Calling this method on a class X
131        will yield test cases from all classes derived from X.
132        """
133        if issubclass(cls, BaseTest) and not inspect.isabstract(cls):
134            #pylint: disable=no-member
135            yield from cls.generate_function_tests()
136        for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
137            yield from subclass.generate_tests()
138
139
140class TestGenerator:
141    """Generate test cases and write to data files."""
142    def __init__(self, options) -> None:
143        self.test_suite_directory = options.directory
144        # Update `targets` with an entry for each child class of BaseTarget.
145        # Each entry represents a file generated by the BaseTarget framework,
146        # and enables generating the .data files using the CLI.
147        self.targets.update({
148            subclass.target_basename: subclass.generate_tests
149            for subclass in BaseTarget.__subclasses__()
150            if subclass.target_basename
151        })
152
153    def filename_for(self, basename: str) -> str:
154        """The location of the data file with the specified base name."""
155        return posixpath.join(self.test_suite_directory, basename + '.data')
156
157    def write_test_data_file(self, basename: str,
158                             test_cases: Iterable[test_case.TestCase]) -> None:
159        """Write the test cases to a .data file.
160
161        The output file is ``basename + '.data'`` in the test suite directory.
162        """
163        filename = self.filename_for(basename)
164        test_case.write_data_file(filename, test_cases)
165
166    # Note that targets whose names contain 'test_format' have their content
167    # validated by `abi_check.py`.
168    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
169
170    def generate_target(self, name: str, *target_args) -> None:
171        """Generate cases and write to data file for a target.
172
173        For target callables which require arguments, override this function
174        and pass these arguments using super() (see PSATestGenerator).
175        """
176        test_cases = self.targets[name](*target_args)
177        self.write_test_data_file(name, test_cases)
178
179def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
180    """Command line entry point."""
181    parser = argparse.ArgumentParser(description=description)
182    parser.add_argument('--list', action='store_true',
183                        help='List available targets and exit')
184    parser.add_argument('--list-for-cmake', action='store_true',
185                        help='Print \';\'-separated list of available targets and exit')
186    # If specified explicitly, this option may be a path relative to the
187    # current directory when the script is invoked. The default value
188    # is relative to the mbedtls root, which we don't know yet. So we
189    # can't set a string as the default value here.
190    parser.add_argument('--directory', metavar='DIR',
191                        help='Output directory (default: tests/suites)')
192    parser.add_argument('targets', nargs='*', metavar='TARGET',
193                        help='Target file to generate (default: all; "-": none)')
194    options = parser.parse_args(args)
195
196    # Change to the mbedtls root, to keep things simple. But first, adjust
197    # command line options that might be relative paths.
198    if options.directory is None:
199        options.directory = 'tests/suites'
200    else:
201        options.directory = os.path.abspath(options.directory)
202    build_tree.chdir_to_root()
203
204    generator = generator_class(options)
205    if options.list:
206        for name in sorted(generator.targets):
207            print(generator.filename_for(name))
208        return
209    # List in a cmake list format (i.e. ';'-separated)
210    if options.list_for_cmake:
211        print(';'.join(generator.filename_for(name)
212                       for name in sorted(generator.targets)), end='')
213        return
214    if options.targets:
215        # Allow "-" as a special case so you can run
216        # ``generate_xxx_tests.py - $targets`` and it works uniformly whether
217        # ``$targets`` is empty or not.
218        options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
219                           for target in options.targets
220                           if target != '-']
221    else:
222        options.targets = sorted(generator.targets)
223    for target in options.targets:
224        generator.generate_target(target)
225