1#!/usr/bin/env python3
2
3"""Test helper for the Mbed TLS configuration file tool
4
5Run config.py with various parameters and write the results to files.
6
7This is a harness to help regression testing, not a functional tester.
8Sample usage:
9
10    test_config_script.py -d old
11    ## Modify config.py and/or mbedtls_config.h ##
12    test_config_script.py -d new
13    diff -ru old new
14"""
15
16## Copyright The Mbed TLS Contributors
17## SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
18##
19
20import argparse
21import glob
22import os
23import re
24import shutil
25import subprocess
26
27OUTPUT_FILE_PREFIX = 'config-'
28
29def output_file_name(directory, stem, extension):
30    return os.path.join(directory,
31                        '{}{}.{}'.format(OUTPUT_FILE_PREFIX,
32                                         stem, extension))
33
34def cleanup_directory(directory):
35    """Remove old output files."""
36    for extension in []:
37        pattern = output_file_name(directory, '*', extension)
38        filenames = glob.glob(pattern)
39        for filename in filenames:
40            os.remove(filename)
41
42def prepare_directory(directory):
43    """Create the output directory if it doesn't exist yet.
44
45    If there are old output files, remove them.
46    """
47    if os.path.exists(directory):
48        cleanup_directory(directory)
49    else:
50        os.makedirs(directory)
51
52def guess_presets_from_help(help_text):
53    """Figure out what presets the script supports.
54
55    help_text should be the output from running the script with --help.
56    """
57    # Try the output format from config.py
58    hits = re.findall(r'\{([-\w,]+)\}', help_text)
59    for hit in hits:
60        words = set(hit.split(','))
61        if 'get' in words and 'set' in words and 'unset' in words:
62            words.remove('get')
63            words.remove('set')
64            words.remove('unset')
65            return words
66    # Try the output format from config.pl
67    hits = re.findall(r'\n +([-\w]+) +- ', help_text)
68    if hits:
69        return hits
70    raise Exception("Unable to figure out supported presets. Pass the '-p' option.")
71
72def list_presets(options):
73    """Return the list of presets to test.
74
75    The list is taken from the command line if present, otherwise it is
76    extracted from running the config script with --help.
77    """
78    if options.presets:
79        return re.split(r'[ ,]+', options.presets)
80    else:
81        help_text = subprocess.run([options.script, '--help'],
82                                   check=False, # config.pl --help returns 255
83                                   stdout=subprocess.PIPE,
84                                   stderr=subprocess.STDOUT).stdout
85        return guess_presets_from_help(help_text.decode('ascii'))
86
87def run_one(options, args, stem_prefix='', input_file=None):
88    """Run the config script with the given arguments.
89
90    Take the original content from input_file if specified, defaulting
91    to options.input_file if input_file is None.
92
93    Write the following files, where xxx contains stem_prefix followed by
94    a filename-friendly encoding of args:
95    * config-xxx.h: modified file.
96    * config-xxx.out: standard output.
97    * config-xxx.err: standard output.
98    * config-xxx.status: exit code.
99
100    Return ("xxx+", "path/to/config-xxx.h") which can be used as
101    stem_prefix and input_file to call this function again with new args.
102    """
103    if input_file is None:
104        input_file = options.input_file
105    stem = stem_prefix + '-'.join(args)
106    data_filename = output_file_name(options.output_directory, stem, 'h')
107    stdout_filename = output_file_name(options.output_directory, stem, 'out')
108    stderr_filename = output_file_name(options.output_directory, stem, 'err')
109    status_filename = output_file_name(options.output_directory, stem, 'status')
110    shutil.copy(input_file, data_filename)
111    # Pass only the file basename, not the full path, to avoid getting the
112    # directory name in error messages, which would make comparisons
113    # between output directories more difficult.
114    cmd = [os.path.abspath(options.script),
115           '-f', os.path.basename(data_filename)]
116    with open(stdout_filename, 'wb') as out:
117        with open(stderr_filename, 'wb') as err:
118            status = subprocess.call(cmd + args,
119                                     cwd=options.output_directory,
120                                     stdin=subprocess.DEVNULL,
121                                     stdout=out, stderr=err)
122    with open(status_filename, 'w') as status_file:
123        status_file.write('{}\n'.format(status))
124    return stem + "+", data_filename
125
126### A list of symbols to test with.
127### This script currently tests what happens when you change a symbol from
128### having a value to not having a value or vice versa. This is not
129### necessarily useful behavior, and we may not consider it a bug if
130### config.py stops handling that case correctly.
131TEST_SYMBOLS = [
132    'CUSTOM_SYMBOL', # does not exist
133    'MBEDTLS_AES_C', # set, no value
134    'MBEDTLS_MPI_MAX_SIZE', # unset, has a value
135    'MBEDTLS_NO_UDBL_DIVISION', # unset, in "System support"
136    'MBEDTLS_PLATFORM_ZEROIZE_ALT', # unset, in "Customisation configuration options"
137]
138
139def run_all(options):
140    """Run all the command lines to test."""
141    presets = list_presets(options)
142    for preset in presets:
143        run_one(options, [preset])
144    for symbol in TEST_SYMBOLS:
145        run_one(options, ['get', symbol])
146        (stem, filename) = run_one(options, ['set', symbol])
147        run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename)
148        run_one(options, ['--force', 'set', symbol])
149        (stem, filename) = run_one(options, ['set', symbol, 'value'])
150        run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename)
151        run_one(options, ['--force', 'set', symbol, 'value'])
152        run_one(options, ['unset', symbol])
153
154def main():
155    """Command line entry point."""
156    parser = argparse.ArgumentParser(description=__doc__,
157                                     formatter_class=argparse.RawDescriptionHelpFormatter)
158    parser.add_argument('-d', metavar='DIR',
159                        dest='output_directory', required=True,
160                        help="""Output directory.""")
161    parser.add_argument('-f', metavar='FILE',
162                        dest='input_file', default='include/mbedtls/mbedtls_config.h',
163                        help="""Config file (default: %(default)s).""")
164    parser.add_argument('-p', metavar='PRESET,...',
165                        dest='presets',
166                        help="""Presets to test (default: guessed from --help).""")
167    parser.add_argument('-s', metavar='FILE',
168                        dest='script', default='scripts/config.py',
169                        help="""Configuration script (default: %(default)s).""")
170    options = parser.parse_args()
171    prepare_directory(options.output_directory)
172    run_all(options)
173
174if __name__ == '__main__':
175    main()
176