1#!/usr/bin/env python3
2"""Run the PSA Crypto API compliance test suite.
3Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
4then compile and run the test suite. The clone is stored at <repository root>/psa-arch-tests.
5Known defects in either the test suite or mbedtls / TF-PSA-Crypto - identified by their test
6number - are ignored, while unexpected failures AND successes are reported as errors, to help
7keep the list of known defects as up to date as possible.
8"""
9
10# Copyright The Mbed TLS Contributors
11# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
12
13import argparse
14import os
15import re
16import shutil
17import subprocess
18import sys
19from typing import List
20
21#pylint: disable=unused-import
22import scripts_path
23from mbedtls_dev import build_tree
24
25# PSA Compliance tests we expect to fail due to known defects in Mbed TLS /
26# TF-PSA-Crypto (or the test suite).
27# The test numbers correspond to the numbers used by the console output of the test suite.
28# Test number 2xx corresponds to the files in the folder
29# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
30EXPECTED_FAILURES = {} # type: dict
31
32PSA_ARCH_TESTS_REPO = 'https://github.com/ARM-software/psa-arch-tests.git'
33PSA_ARCH_TESTS_REF = 'v23.06_API1.5_ADAC_EAC'
34
35#pylint: disable=too-many-branches,too-many-statements,too-many-locals
36def main(library_build_dir: str):
37    root_dir = os.getcwd()
38
39    in_tf_psa_crypto_repo = build_tree.looks_like_tf_psa_crypto_root(root_dir)
40
41    crypto_name = build_tree.crypto_library_filename(root_dir)
42    library_subdir = build_tree.crypto_core_directory(root_dir, relative=True)
43
44    crypto_lib_filename = (library_build_dir + '/' +
45                           library_subdir + '/' +
46                           'lib' + crypto_name + '.a')
47
48    if not os.path.exists(crypto_lib_filename):
49        #pylint: disable=bad-continuation
50        subprocess.check_call([
51            'cmake', '.',
52                     '-GUnix Makefiles',
53                     '-B' + library_build_dir
54        ])
55        subprocess.check_call(['cmake', '--build', library_build_dir,
56                               '--target', crypto_name])
57
58    psa_arch_tests_dir = 'psa-arch-tests'
59    os.makedirs(psa_arch_tests_dir, exist_ok=True)
60    try:
61        os.chdir(psa_arch_tests_dir)
62
63        # Reuse existing local clone
64        subprocess.check_call(['git', 'init'])
65        subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
66        subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
67
68        build_dir = 'api-tests/build'
69        try:
70            shutil.rmtree(build_dir)
71        except FileNotFoundError:
72            pass
73        os.mkdir(build_dir)
74        os.chdir(build_dir)
75
76        extra_includes = (';{}/drivers/builtin/include'.format(root_dir)
77                          if in_tf_psa_crypto_repo else '')
78
79        #pylint: disable=bad-continuation
80        subprocess.check_call([
81            'cmake', '..',
82                     '-GUnix Makefiles',
83                     '-DTARGET=tgt_dev_apis_stdc',
84                     '-DTOOLCHAIN=HOST_GCC',
85                     '-DSUITE=CRYPTO',
86                     '-DPSA_CRYPTO_LIB_FILENAME={}/{}'.format(root_dir,
87                                                              crypto_lib_filename),
88                     ('-DPSA_INCLUDE_PATHS={}/include' + extra_includes).format(root_dir)
89        ])
90        subprocess.check_call(['cmake', '--build', '.'])
91
92        proc = subprocess.Popen(['./psa-arch-tests-crypto'],
93                                bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
94
95        test_re = re.compile(
96            '^TEST: (?P<test_num>[0-9]*)|'
97            '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
98        )
99        test = -1
100        unexpected_successes = set(EXPECTED_FAILURES)
101        expected_failures = [] # type: List[int]
102        unexpected_failures = [] # type: List[int]
103        if proc.stdout is None:
104            return 1
105
106        for line in proc.stdout:
107            print(line, end='')
108            match = test_re.match(line)
109            if match is not None:
110                groupdict = match.groupdict()
111                test_num = groupdict['test_num']
112                if test_num is not None:
113                    test = int(test_num)
114                elif groupdict['test_result'] == 'FAILED':
115                    try:
116                        unexpected_successes.remove(test)
117                        expected_failures.append(test)
118                        print('Expected failure, ignoring')
119                    except KeyError:
120                        unexpected_failures.append(test)
121                        print('ERROR: Unexpected failure')
122                elif test in unexpected_successes:
123                    print('ERROR: Unexpected success')
124        proc.wait()
125
126        print()
127        print('***** test_psa_compliance.py report ******')
128        print()
129        print('Expected failures:', ', '.join(str(i) for i in expected_failures))
130        print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
131        print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
132        print()
133        if unexpected_successes or unexpected_failures:
134            if unexpected_successes:
135                print('Unexpected successes encountered.')
136                print('Please remove the corresponding tests from '
137                      'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
138                print()
139            print('FAILED')
140            return 1
141        else:
142            print('SUCCESS')
143            return 0
144    finally:
145        os.chdir(root_dir)
146
147if __name__ == '__main__':
148    BUILD_DIR = 'out_of_source_build'
149
150    # pylint: disable=invalid-name
151    parser = argparse.ArgumentParser()
152    parser.add_argument('--build-dir', nargs=1,
153                        help='path to Mbed TLS / TF-PSA-Crypto build directory')
154    args = parser.parse_args()
155
156    if args.build_dir is not None:
157        BUILD_DIR = args.build_dir[0]
158
159    sys.exit(main(BUILD_DIR))
160