1#!/usr/bin/env python3
2"""Describe the test coverage of PSA functions in terms of return statuses.
3
41. Build Mbed TLS with -DRECORD_PSA_STATUS_COVERAGE_LOG
52. Run psa_collect_statuses.py
6
7The output is a series of line of the form "psa_foo PSA_ERROR_XXX". Each
8function/status combination appears only once.
9
10This script must be run from the top of an Mbed TLS source tree.
11The build command is "make -DRECORD_PSA_STATUS_COVERAGE_LOG", which is
12only supported with make (as opposed to CMake or other build methods).
13"""
14
15# Copyright The Mbed TLS Contributors
16# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
17
18import argparse
19import os
20import subprocess
21import sys
22
23DEFAULT_STATUS_LOG_FILE = 'tests/statuses.log'
24DEFAULT_PSA_CONSTANT_NAMES = 'programs/psa/psa_constant_names'
25
26class Statuses:
27    """Information about observed return statues of API functions."""
28
29    def __init__(self):
30        self.functions = {}
31        self.codes = set()
32        self.status_names = {}
33
34    def collect_log(self, log_file_name):
35        """Read logs from RECORD_PSA_STATUS_COVERAGE_LOG.
36
37        Read logs produced by running Mbed TLS test suites built with
38        -DRECORD_PSA_STATUS_COVERAGE_LOG.
39        """
40        with open(log_file_name) as log:
41            for line in log:
42                value, function, tail = line.split(':', 2)
43                if function not in self.functions:
44                    self.functions[function] = {}
45                fdata = self.functions[function]
46                if value not in self.functions[function]:
47                    fdata[value] = []
48                fdata[value].append(tail)
49                self.codes.add(int(value))
50
51    def get_constant_names(self, psa_constant_names):
52        """Run psa_constant_names to obtain names for observed numerical values."""
53        values = [str(value) for value in self.codes]
54        cmd = [psa_constant_names, 'status'] + values
55        output = subprocess.check_output(cmd).decode('ascii')
56        for value, name in zip(values, output.rstrip().split('\n')):
57            self.status_names[value] = name
58
59    def report(self):
60        """Report observed return values for each function.
61
62        The report is a series of line of the form "psa_foo PSA_ERROR_XXX".
63        """
64        for function in sorted(self.functions.keys()):
65            fdata = self.functions[function]
66            names = [self.status_names[value] for value in fdata.keys()]
67            for name in sorted(names):
68                sys.stdout.write('{} {}\n'.format(function, name))
69
70def collect_status_logs(options):
71    """Build and run unit tests and report observed function return statuses.
72
73    Build Mbed TLS with -DRECORD_PSA_STATUS_COVERAGE_LOG, run the
74    test suites and display information about observed return statuses.
75    """
76    rebuilt = False
77    if not options.use_existing_log and os.path.exists(options.log_file):
78        os.remove(options.log_file)
79    if not os.path.exists(options.log_file):
80        if options.clean_before:
81            subprocess.check_call(['make', 'clean'],
82                                  cwd='tests',
83                                  stdout=sys.stderr)
84        with open(os.devnull, 'w') as devnull:
85            make_q_ret = subprocess.call(['make', '-q', 'lib', 'tests'],
86                                         stdout=devnull, stderr=devnull)
87        if make_q_ret != 0:
88            subprocess.check_call(['make', 'RECORD_PSA_STATUS_COVERAGE_LOG=1'],
89                                  stdout=sys.stderr)
90            rebuilt = True
91        subprocess.check_call(['make', 'test'],
92                              stdout=sys.stderr)
93    data = Statuses()
94    data.collect_log(options.log_file)
95    data.get_constant_names(options.psa_constant_names)
96    if rebuilt and options.clean_after:
97        subprocess.check_call(['make', 'clean'],
98                              cwd='tests',
99                              stdout=sys.stderr)
100    return data
101
102def main():
103    parser = argparse.ArgumentParser(description=globals()['__doc__'])
104    parser.add_argument('--clean-after',
105                        action='store_true',
106                        help='Run "make clean" after rebuilding')
107    parser.add_argument('--clean-before',
108                        action='store_true',
109                        help='Run "make clean" before regenerating the log file)')
110    parser.add_argument('--log-file', metavar='FILE',
111                        default=DEFAULT_STATUS_LOG_FILE,
112                        help='Log file location (default: {})'.format(
113                            DEFAULT_STATUS_LOG_FILE
114                        ))
115    parser.add_argument('--psa-constant-names', metavar='PROGRAM',
116                        default=DEFAULT_PSA_CONSTANT_NAMES,
117                        help='Path to psa_constant_names (default: {})'.format(
118                            DEFAULT_PSA_CONSTANT_NAMES
119                        ))
120    parser.add_argument('--use-existing-log', '-e',
121                        action='store_true',
122                        help='Don\'t regenerate the log file if it exists')
123    options = parser.parse_args()
124    data = collect_status_logs(options)
125    data.report()
126
127if __name__ == '__main__':
128    main()
129