1#!/usr/bin/env python3
2"""Run the PSA Crypto API compliance test suite.
3Clone the repo and check out the commit specified by PSA_ARCH_TEST_REPO and PSA_ARCH_TEST_REF,
4then compile and run the test suite. The clone is stored at <Mbed TLS root>/psa-arch-tests.
5Known defects in either the test suite or mbedtls - identified by their test number - are ignored,
6while unexpected failures AND successes are reported as errors,
7to help keep the list of known defects as up to date as possible.
8"""
9
10# Copyright The Mbed TLS Contributors
11# SPDX-License-Identifier: Apache-2.0
12#
13# Licensed under the Apache License, Version 2.0 (the "License"); you may
14# not use this file except in compliance with the License.
15# You may obtain a copy of the License at
16#
17# http://www.apache.org/licenses/LICENSE-2.0
18#
19# Unless required by applicable law or agreed to in writing, software
20# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
21# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22# See the License for the specific language governing permissions and
23# limitations under the License.
24
25import os
26import re
27import shutil
28import subprocess
29import sys
30
31# PSA Compliance tests we expect to fail due to known defects in Mbed TLS (or the test suite)
32# The test numbers correspond to the numbers used by the console output of the test suite.
33# Test number 2xx corresponds to the files in the folder
34# psa-arch-tests/api-tests/dev_apis/crypto/test_c0xx
35EXPECTED_FAILURES = {
36    # psa_hash_suspend() and psa_hash_resume() are not supported.
37    # - Tracked in issue #3274
38    262, 263
39}
40
41# We currently use a fork of ARM-software/psa-arch-tests, with a couple of downstream patches
42# that allow it to build with MbedTLS 3, and fixes a couple of issues in the compliance test suite.
43# These fixes allow the tests numbered 216, 248 and 249 to complete successfully.
44#
45# Once all the fixes are upstreamed, this fork should be replaced with an upstream commit/tag.
46# - Tracked in issue #5145
47#
48# Web URL: https://github.com/bensze01/psa-arch-tests/tree/fixes-for-mbedtls-3
49PSA_ARCH_TESTS_REPO = 'https://github.com/bensze01/psa-arch-tests.git'
50PSA_ARCH_TESTS_REF = 'fix-pr-5736'
51
52#pylint: disable=too-many-branches,too-many-statements
53def main():
54    mbedtls_dir = os.getcwd()
55
56    if not os.path.exists('library/libmbedcrypto.a'):
57        subprocess.check_call(['make', '-C', 'library', 'libmbedcrypto.a'])
58
59    psa_arch_tests_dir = 'psa-arch-tests'
60    os.makedirs(psa_arch_tests_dir, exist_ok=True)
61    try:
62        os.chdir(psa_arch_tests_dir)
63
64        # Reuse existing local clone
65        subprocess.check_call(['git', 'init'])
66        subprocess.check_call(['git', 'fetch', PSA_ARCH_TESTS_REPO, PSA_ARCH_TESTS_REF])
67        subprocess.check_call(['git', 'checkout', 'FETCH_HEAD'])
68
69        build_dir = 'api-tests/build'
70        try:
71            shutil.rmtree(build_dir)
72        except FileNotFoundError:
73            pass
74        os.mkdir(build_dir)
75        os.chdir(build_dir)
76
77        #pylint: disable=bad-continuation
78        subprocess.check_call([
79            'cmake', '..',
80                     '-GUnix Makefiles',
81                     '-DTARGET=tgt_dev_apis_stdc',
82                     '-DTOOLCHAIN=HOST_GCC',
83                     '-DSUITE=CRYPTO',
84                     '-DPSA_CRYPTO_LIB_FILENAME={}/library/libmbedcrypto.a'.format(mbedtls_dir),
85                     '-DPSA_INCLUDE_PATHS={}/include'.format(mbedtls_dir)
86        ])
87        subprocess.check_call(['cmake', '--build', '.'])
88
89        proc = subprocess.Popen(['./psa-arch-tests-crypto'],
90                                bufsize=1, stdout=subprocess.PIPE, universal_newlines=True)
91
92        test_re = re.compile(
93            '^TEST: (?P<test_num>[0-9]*)|'
94            '^TEST RESULT: (?P<test_result>FAILED|PASSED)'
95        )
96        test = -1
97        unexpected_successes = set(EXPECTED_FAILURES)
98        expected_failures = []
99        unexpected_failures = []
100        for line in proc.stdout:
101            print(line, end='')
102            match = test_re.match(line)
103            if match is not None:
104                groupdict = match.groupdict()
105                test_num = groupdict['test_num']
106                if test_num is not None:
107                    test = int(test_num)
108                elif groupdict['test_result'] == 'FAILED':
109                    try:
110                        unexpected_successes.remove(test)
111                        expected_failures.append(test)
112                        print('Expected failure, ignoring')
113                    except KeyError:
114                        unexpected_failures.append(test)
115                        print('ERROR: Unexpected failure')
116                elif test in unexpected_successes:
117                    print('ERROR: Unexpected success')
118        proc.wait()
119
120        print()
121        print('***** test_psa_compliance.py report ******')
122        print()
123        print('Expected failures:', ', '.join(str(i) for i in expected_failures))
124        print('Unexpected failures:', ', '.join(str(i) for i in unexpected_failures))
125        print('Unexpected successes:', ', '.join(str(i) for i in sorted(unexpected_successes)))
126        print()
127        if unexpected_successes or unexpected_failures:
128            if unexpected_successes:
129                print('Unexpected successes encountered.')
130                print('Please remove the corresponding tests from '
131                      'EXPECTED_FAILURES in tests/scripts/compliance_test.py')
132                print()
133            print('FAILED')
134            return 1
135        else:
136            print('SUCCESS')
137            return 0
138    finally:
139        os.chdir(mbedtls_dir)
140
141if __name__ == '__main__':
142    sys.exit(main())
143