1"""Common code for test data generation.
2
3This module defines classes that are of general use to automatically
4generate .data files for unit tests, as well as a main function.
5
6These are used both by generate_psa_tests.py and generate_bignum_tests.py.
7"""
8
9# Copyright The Mbed TLS Contributors
10# SPDX-License-Identifier: Apache-2.0
11#
12# Licensed under the Apache License, Version 2.0 (the "License"); you may
13# not use this file except in compliance with the License.
14# You may obtain a copy of the License at
15#
16# http://www.apache.org/licenses/LICENSE-2.0
17#
18# Unless required by applicable law or agreed to in writing, software
19# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
20# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21# See the License for the specific language governing permissions and
22# limitations under the License.
23
24import argparse
25import os
26import posixpath
27import re
28import inspect
29
30from abc import ABCMeta, abstractmethod
31from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar
32
33from . import build_tree
34from . import test_case
35
36T = TypeVar('T') #pylint: disable=invalid-name
37
38
39class BaseTest(metaclass=ABCMeta):
40    """Base class for test case generation.
41
42    Attributes:
43        count: Counter for test cases from this class.
44        case_description: Short description of the test case. This may be
45            automatically generated using the class, or manually set.
46        dependencies: A list of dependencies required for the test case.
47        show_test_count: Toggle for inclusion of `count` in the test description.
48        test_function: Test function which the class generates cases for.
49        test_name: A common name or description of the test function. This can
50            be `test_function`, a clearer equivalent, or a short summary of the
51            test function's purpose.
52    """
53    count = 0
54    case_description = ""
55    dependencies = [] # type: List[str]
56    show_test_count = True
57    test_function = ""
58    test_name = ""
59
60    def __new__(cls, *args, **kwargs):
61        # pylint: disable=unused-argument
62        cls.count += 1
63        return super().__new__(cls)
64
65    @abstractmethod
66    def arguments(self) -> List[str]:
67        """Get the list of arguments for the test case.
68
69        Override this method to provide the list of arguments required for
70        the `test_function`.
71
72        Returns:
73            List of arguments required for the test function.
74        """
75        raise NotImplementedError
76
77    def description(self) -> str:
78        """Create a test case description.
79
80        Creates a description of the test case, including a name for the test
81        function, an optional case count, and a description of the specific
82        test case. This should inform a reader what is being tested, and
83        provide context for the test case.
84
85        Returns:
86            Description for the test case.
87        """
88        if self.show_test_count:
89            return "{} #{} {}".format(
90                self.test_name, self.count, self.case_description
91                ).strip()
92        else:
93            return "{} {}".format(self.test_name, self.case_description).strip()
94
95
96    def create_test_case(self) -> test_case.TestCase:
97        """Generate TestCase from the instance."""
98        tc = test_case.TestCase()
99        tc.set_description(self.description())
100        tc.set_function(self.test_function)
101        tc.set_arguments(self.arguments())
102        tc.set_dependencies(self.dependencies)
103
104        return tc
105
106    @classmethod
107    @abstractmethod
108    def generate_function_tests(cls) -> Iterator[test_case.TestCase]:
109        """Generate test cases for the class test function.
110
111        This will be called in classes where `test_function` is set.
112        Implementations should yield TestCase objects, by creating instances
113        of the class with appropriate input data, and then calling
114        `create_test_case()` on each.
115        """
116        raise NotImplementedError
117
118
119class BaseTarget:
120    #pylint: disable=too-few-public-methods
121    """Base target for test case generation.
122
123    Child classes of this class represent an output file, and can be referred
124    to as file targets. These indicate where test cases will be written to for
125    all subclasses of the file target, which is set by `target_basename`.
126
127    Attributes:
128        target_basename: Basename of file to write generated tests to. This
129            should be specified in a child class of BaseTarget.
130    """
131    target_basename = ""
132
133    @classmethod
134    def generate_tests(cls) -> Iterator[test_case.TestCase]:
135        """Generate test cases for the class and its subclasses.
136
137        In classes with `test_function` set, `generate_function_tests()` is
138        called to generate test cases first.
139
140        In all classes, this method will iterate over its subclasses, and
141        yield from `generate_tests()` in each. Calling this method on a class X
142        will yield test cases from all classes derived from X.
143        """
144        if issubclass(cls, BaseTest) and not inspect.isabstract(cls):
145            #pylint: disable=no-member
146            yield from cls.generate_function_tests()
147        for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__):
148            yield from subclass.generate_tests()
149
150
151class TestGenerator:
152    """Generate test cases and write to data files."""
153    def __init__(self, options) -> None:
154        self.test_suite_directory = options.directory
155        # Update `targets` with an entry for each child class of BaseTarget.
156        # Each entry represents a file generated by the BaseTarget framework,
157        # and enables generating the .data files using the CLI.
158        self.targets.update({
159            subclass.target_basename: subclass.generate_tests
160            for subclass in BaseTarget.__subclasses__()
161            if subclass.target_basename
162        })
163
164    def filename_for(self, basename: str) -> str:
165        """The location of the data file with the specified base name."""
166        return posixpath.join(self.test_suite_directory, basename + '.data')
167
168    def write_test_data_file(self, basename: str,
169                             test_cases: Iterable[test_case.TestCase]) -> None:
170        """Write the test cases to a .data file.
171
172        The output file is ``basename + '.data'`` in the test suite directory.
173        """
174        filename = self.filename_for(basename)
175        test_case.write_data_file(filename, test_cases)
176
177    # Note that targets whose names contain 'test_format' have their content
178    # validated by `abi_check.py`.
179    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
180
181    def generate_target(self, name: str, *target_args) -> None:
182        """Generate cases and write to data file for a target.
183
184        For target callables which require arguments, override this function
185        and pass these arguments using super() (see PSATestGenerator).
186        """
187        test_cases = self.targets[name](*target_args)
188        self.write_test_data_file(name, test_cases)
189
190def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
191    """Command line entry point."""
192    parser = argparse.ArgumentParser(description=description)
193    parser.add_argument('--list', action='store_true',
194                        help='List available targets and exit')
195    parser.add_argument('--list-for-cmake', action='store_true',
196                        help='Print \';\'-separated list of available targets and exit')
197    # If specified explicitly, this option may be a path relative to the
198    # current directory when the script is invoked. The default value
199    # is relative to the mbedtls root, which we don't know yet. So we
200    # can't set a string as the default value here.
201    parser.add_argument('--directory', metavar='DIR',
202                        help='Output directory (default: tests/suites)')
203    parser.add_argument('targets', nargs='*', metavar='TARGET',
204                        help='Target file to generate (default: all; "-": none)')
205    options = parser.parse_args(args)
206
207    # Change to the mbedtls root, to keep things simple. But first, adjust
208    # command line options that might be relative paths.
209    if options.directory is None:
210        options.directory = 'tests/suites'
211    else:
212        options.directory = os.path.abspath(options.directory)
213    build_tree.chdir_to_root()
214
215    generator = generator_class(options)
216    if options.list:
217        for name in sorted(generator.targets):
218            print(generator.filename_for(name))
219        return
220    # List in a cmake list format (i.e. ';'-separated)
221    if options.list_for_cmake:
222        print(';'.join(generator.filename_for(name)
223                       for name in sorted(generator.targets)), end='')
224        return
225    if options.targets:
226        # Allow "-" as a special case so you can run
227        # ``generate_xxx_tests.py - $targets`` and it works uniformly whether
228        # ``$targets`` is empty or not.
229        options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target))
230                           for target in options.targets
231                           if target != '-']
232    else:
233        options.targets = sorted(generator.targets)
234    for target in options.targets:
235        generator.generate_target(target)
236