1#!/usr/bin/env python3 2 3"""Analyze the test outcomes from a full CI run. 4 5This script can also run on outcomes from a partial run, but the results are 6less likely to be useful. 7""" 8 9import argparse 10import sys 11import traceback 12import re 13import subprocess 14import os 15import typing 16 17import check_test_cases 18 19 20# `ComponentOutcomes` is a named tuple which is defined as: 21# ComponentOutcomes( 22# successes = { 23# "<suite_case>", 24# ... 25# }, 26# failures = { 27# "<suite_case>", 28# ... 29# } 30# ) 31# suite_case = "<suite>;<case>" 32ComponentOutcomes = typing.NamedTuple('ComponentOutcomes', 33 [('successes', typing.Set[str]), 34 ('failures', typing.Set[str])]) 35 36# `Outcomes` is a representation of the outcomes file, 37# which defined as: 38# Outcomes = { 39# "<component>": ComponentOutcomes, 40# ... 41# } 42Outcomes = typing.Dict[str, ComponentOutcomes] 43 44 45class Results: 46 """Process analysis results.""" 47 48 def __init__(self): 49 self.error_count = 0 50 self.warning_count = 0 51 52 def new_section(self, fmt, *args, **kwargs): 53 self._print_line('\n*** ' + fmt + ' ***\n', *args, **kwargs) 54 55 def info(self, fmt, *args, **kwargs): 56 self._print_line('Info: ' + fmt, *args, **kwargs) 57 58 def error(self, fmt, *args, **kwargs): 59 self.error_count += 1 60 self._print_line('Error: ' + fmt, *args, **kwargs) 61 62 def warning(self, fmt, *args, **kwargs): 63 self.warning_count += 1 64 self._print_line('Warning: ' + fmt, *args, **kwargs) 65 66 @staticmethod 67 def _print_line(fmt, *args, **kwargs): 68 sys.stderr.write((fmt + '\n').format(*args, **kwargs)) 69 70def execute_reference_driver_tests(results: Results, ref_component: str, driver_component: str, \ 71 outcome_file: str) -> None: 72 """Run the tests specified in ref_component and driver_component. Results 73 are stored in the output_file and they will be used for the following 74 coverage analysis""" 75 results.new_section("Test {} and {}", ref_component, driver_component) 76 77 shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \ 78 " " + ref_component + " " + driver_component 79 results.info("Running: {}", shell_command) 80 ret_val = subprocess.run(shell_command.split(), check=False).returncode 81 82 if ret_val != 0: 83 results.error("failed to run reference/driver components") 84 85def analyze_coverage(results: Results, outcomes: Outcomes, 86 allow_list: typing.List[str], full_coverage: bool) -> None: 87 """Check that all available test cases are executed at least once.""" 88 # Make sure that the generated data files are present (and up-to-date). 89 # This allows analyze_outcomes.py to run correctly on a fresh Git 90 # checkout. 91 cp = subprocess.run(['make', 'generated_files'], 92 cwd='tests', 93 stdout=subprocess.PIPE, stderr=subprocess.STDOUT, 94 check=False) 95 if cp.returncode != 0: 96 sys.stderr.write(cp.stdout.decode('utf-8')) 97 results.error("Failed \"make generated_files\" in tests. " 98 "Coverage analysis may be incorrect.") 99 available = check_test_cases.collect_available_test_cases() 100 for suite_case in available: 101 hit = any(suite_case in comp_outcomes.successes or 102 suite_case in comp_outcomes.failures 103 for comp_outcomes in outcomes.values()) 104 105 if not hit and suite_case not in allow_list: 106 if full_coverage: 107 results.error('Test case not executed: {}', suite_case) 108 else: 109 results.warning('Test case not executed: {}', suite_case) 110 elif hit and suite_case in allow_list: 111 # Test Case should be removed from the allow list. 112 if full_coverage: 113 results.error('Allow listed test case was executed: {}', suite_case) 114 else: 115 results.warning('Allow listed test case was executed: {}', suite_case) 116 117def name_matches_pattern(name: str, str_or_re) -> bool: 118 """Check if name matches a pattern, that may be a string or regex. 119 - If the pattern is a string, name must be equal to match. 120 - If the pattern is a regex, name must fully match. 121 """ 122 # The CI's python is too old for re.Pattern 123 #if isinstance(str_or_re, re.Pattern): 124 if not isinstance(str_or_re, str): 125 return str_or_re.fullmatch(name) is not None 126 else: 127 return str_or_re == name 128 129def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, 130 component_ref: str, component_driver: str, 131 ignored_suites: typing.List[str], ignored_tests=None) -> None: 132 """Check that all tests passing in the reference component are also 133 passing in the corresponding driver component. 134 Skip: 135 - full test suites provided in ignored_suites list 136 - only some specific test inside a test suite, for which the corresponding 137 output string is provided 138 """ 139 ref_outcomes = outcomes.get("component_" + component_ref) 140 driver_outcomes = outcomes.get("component_" + component_driver) 141 142 if ref_outcomes is None or driver_outcomes is None: 143 results.error("required components are missing: bad outcome file?") 144 return 145 146 if not ref_outcomes.successes: 147 results.error("no passing test in reference component: bad outcome file?") 148 return 149 150 for suite_case in ref_outcomes.successes: 151 # suite_case is like "test_suite_foo.bar;Description of test case" 152 (full_test_suite, test_string) = suite_case.split(';') 153 test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name 154 155 # Immediately skip fully-ignored test suites 156 if test_suite in ignored_suites or full_test_suite in ignored_suites: 157 continue 158 159 # For ignored test cases inside test suites, just remember and: 160 # don't issue an error if they're skipped with drivers, 161 # but issue an error if they're not (means we have a bad entry). 162 ignored = False 163 for str_or_re in (ignored_tests.get(full_test_suite, []) + 164 ignored_tests.get(test_suite, [])): 165 if name_matches_pattern(test_string, str_or_re): 166 ignored = True 167 168 if not ignored and not suite_case in driver_outcomes.successes: 169 results.error("PASS -> SKIP/FAIL: {}", suite_case) 170 if ignored and suite_case in driver_outcomes.successes: 171 results.error("uselessly ignored: {}", suite_case) 172 173def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None: 174 """Run all analyses on the given outcome collection.""" 175 analyze_coverage(results, outcomes, args['allow_list'], 176 args['full_coverage']) 177 178def read_outcome_file(outcome_file: str) -> Outcomes: 179 """Parse an outcome file and return an outcome collection. 180 """ 181 outcomes = {} 182 with open(outcome_file, 'r', encoding='utf-8') as input_file: 183 for line in input_file: 184 (_platform, component, suite, case, result, _cause) = line.split(';') 185 # Note that `component` is not unique. If a test case passes on Linux 186 # and fails on FreeBSD, it'll end up in both the successes set and 187 # the failures set. 188 suite_case = ';'.join([suite, case]) 189 if component not in outcomes: 190 outcomes[component] = ComponentOutcomes(set(), set()) 191 if result == 'PASS': 192 outcomes[component].successes.add(suite_case) 193 elif result == 'FAIL': 194 outcomes[component].failures.add(suite_case) 195 196 return outcomes 197 198def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None: 199 """Perform coverage analysis.""" 200 results.new_section("Analyze coverage") 201 analyze_outcomes(results, outcomes, args) 202 203def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: 204 """Perform driver vs reference analyze.""" 205 results.new_section("Analyze driver {} vs reference {}", 206 args['component_driver'], args['component_ref']) 207 208 ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] 209 210 analyze_driver_vs_reference(results, outcomes, 211 args['component_ref'], args['component_driver'], 212 ignored_suites, args['ignored_tests']) 213 214# List of tasks with a function that can handle this task and additional arguments if required 215KNOWN_TASKS = { 216 'analyze_coverage': { 217 'test_function': do_analyze_coverage, 218 'args': { 219 'allow_list': [ 220 # Algorithm not supported yet 221 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', 222 # Algorithm not supported yet 223 'test_suite_psa_crypto_metadata;Cipher: XTS', 224 ], 225 'full_coverage': False, 226 } 227 }, 228 # There are 2 options to use analyze_driver_vs_reference_xxx locally: 229 # 1. Run tests and then analysis: 230 # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver> 231 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx 232 # 2. Let this script run both automatically: 233 # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx 234 'analyze_driver_vs_reference_hash': { 235 'test_function': do_analyze_driver_vs_reference, 236 'args': { 237 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', 238 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', 239 'ignored_suites': [ 240 'shax', 'mdx', # the software implementations that are being excluded 241 'md.psa', # purposefully depends on whether drivers are present 242 'psa_crypto_low_hash.generated', # testing the builtins 243 ], 244 'ignored_tests': { 245 'test_suite_config': [ 246 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 247 ], 248 'test_suite_platform': [ 249 # Incompatible with sanitizers (e.g. ASan). If the driver 250 # component uses a sanitizer but the reference component 251 # doesn't, we have a PASS vs SKIP mismatch. 252 'Check mbedtls_calloc overallocation', 253 ], 254 } 255 } 256 }, 257 'analyze_driver_vs_reference_hmac': { 258 'test_function': do_analyze_driver_vs_reference, 259 'args': { 260 'component_ref': 'test_psa_crypto_config_reference_hmac', 261 'component_driver': 'test_psa_crypto_config_accel_hmac', 262 'ignored_suites': [ 263 # These suites require legacy hash support, which is disabled 264 # in the accelerated component. 265 'shax', 'mdx', 266 # This suite tests builtins directly, but these are missing 267 # in the accelerated case. 268 'psa_crypto_low_hash.generated', 269 ], 270 'ignored_tests': { 271 'test_suite_config': [ 272 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 273 re.compile(r'.*\bMBEDTLS_MD_C\b') 274 ], 275 'test_suite_md': [ 276 # Builtin HMAC is not supported in the accelerate component. 277 re.compile('.*HMAC.*'), 278 # Following tests make use of functions which are not available 279 # when MD_C is disabled, as it happens in the accelerated 280 # test component. 281 re.compile('generic .* Hash file .*'), 282 'MD list', 283 ], 284 'test_suite_md.psa': [ 285 # "legacy only" tests require hash algorithms to be NOT 286 # accelerated, but this of course false for the accelerated 287 # test component. 288 re.compile('PSA dispatch .* legacy only'), 289 ], 290 'test_suite_platform': [ 291 # Incompatible with sanitizers (e.g. ASan). If the driver 292 # component uses a sanitizer but the reference component 293 # doesn't, we have a PASS vs SKIP mismatch. 294 'Check mbedtls_calloc overallocation', 295 ], 296 } 297 } 298 }, 299 'analyze_driver_vs_reference_cipher_aead_cmac': { 300 'test_function': do_analyze_driver_vs_reference, 301 'args': { 302 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac', 303 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac', 304 # Modules replaced by drivers. 305 'ignored_suites': [ 306 # low-level (block/stream) cipher modules 307 'aes', 'aria', 'camellia', 'des', 'chacha20', 308 # AEAD modes and CMAC 309 'ccm', 'chachapoly', 'cmac', 'gcm', 310 # The Cipher abstraction layer 311 'cipher', 312 ], 313 'ignored_tests': { 314 'test_suite_config': [ 315 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), 316 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'), 317 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 318 re.compile(r'.*\bMBEDTLS_CIPHER_.*'), 319 ], 320 # PEM decryption is not supported so far. 321 # The rest of PEM (write, unencrypted read) works though. 322 'test_suite_pem': [ 323 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), 324 ], 325 'test_suite_platform': [ 326 # Incompatible with sanitizers (e.g. ASan). If the driver 327 # component uses a sanitizer but the reference component 328 # doesn't, we have a PASS vs SKIP mismatch. 329 'Check mbedtls_calloc overallocation', 330 ], 331 # Following tests depend on AES_C/DES_C but are not about 332 # them really, just need to know some error code is there. 333 'test_suite_error': [ 334 'Low and high error', 335 'Single low error' 336 ], 337 # Similar to test_suite_error above. 338 'test_suite_version': [ 339 'Check for MBEDTLS_AES_C when already present', 340 ], 341 # The en/decryption part of PKCS#12 is not supported so far. 342 # The rest of PKCS#12 (key derivation) works though. 343 'test_suite_pkcs12': [ 344 re.compile(r'PBE Encrypt, .*'), 345 re.compile(r'PBE Decrypt, .*'), 346 ], 347 # The en/decryption part of PKCS#5 is not supported so far. 348 # The rest of PKCS#5 (PBKDF2) works though. 349 'test_suite_pkcs5': [ 350 re.compile(r'PBES2 Encrypt, .*'), 351 re.compile(r'PBES2 Decrypt .*'), 352 ], 353 # Encrypted keys are not supported so far. 354 # pylint: disable=line-too-long 355 'test_suite_pkparse': [ 356 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 357 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', 358 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), 359 ], 360 # Encrypted keys are not supported so far. 361 'ssl-opt': [ 362 'TLS: password protected server key', 363 'TLS: password protected client key', 364 'TLS: password protected server key, two certificates', 365 ], 366 } 367 } 368 }, 369 'analyze_driver_vs_reference_ecp_light_only': { 370 'test_function': do_analyze_driver_vs_reference, 371 'args': { 372 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', 373 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', 374 'ignored_suites': [ 375 # Modules replaced by drivers 376 'ecdsa', 'ecdh', 'ecjpake', 377 ], 378 'ignored_tests': { 379 'test_suite_config': [ 380 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 381 ], 382 'test_suite_platform': [ 383 # Incompatible with sanitizers (e.g. ASan). If the driver 384 # component uses a sanitizer but the reference component 385 # doesn't, we have a PASS vs SKIP mismatch. 386 'Check mbedtls_calloc overallocation', 387 ], 388 # This test wants a legacy function that takes f_rng, p_rng 389 # arguments, and uses legacy ECDSA for that. The test is 390 # really about the wrapper around the PSA RNG, not ECDSA. 391 'test_suite_random': [ 392 'PSA classic wrapper: ECDSA signature (SECP256R1)', 393 ], 394 # In the accelerated test ECP_C is not set (only ECP_LIGHT is) 395 # so we must ignore disparities in the tests for which ECP_C 396 # is required. 397 'test_suite_ecp': [ 398 re.compile(r'ECP check public-private .*'), 399 re.compile(r'ECP calculate public: .*'), 400 re.compile(r'ECP gen keypair .*'), 401 re.compile(r'ECP point muladd .*'), 402 re.compile(r'ECP point multiplication .*'), 403 re.compile(r'ECP test vectors .*'), 404 ], 405 'test_suite_ssl': [ 406 # This deprecated function is only present when ECP_C is On. 407 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 408 ], 409 } 410 } 411 }, 412 'analyze_driver_vs_reference_no_ecp_at_all': { 413 'test_function': do_analyze_driver_vs_reference, 414 'args': { 415 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', 416 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', 417 'ignored_suites': [ 418 # Modules replaced by drivers 419 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 420 ], 421 'ignored_tests': { 422 'test_suite_config': [ 423 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 424 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 425 ], 426 'test_suite_platform': [ 427 # Incompatible with sanitizers (e.g. ASan). If the driver 428 # component uses a sanitizer but the reference component 429 # doesn't, we have a PASS vs SKIP mismatch. 430 'Check mbedtls_calloc overallocation', 431 ], 432 # See ecp_light_only 433 'test_suite_random': [ 434 'PSA classic wrapper: ECDSA signature (SECP256R1)', 435 ], 436 'test_suite_pkparse': [ 437 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED 438 # is automatically enabled in build_info.h (backward compatibility) 439 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a 440 # consequence compressed points are supported in the reference 441 # component but not in the accelerated one, so they should be skipped 442 # while checking driver's coverage. 443 re.compile(r'Parse EC Key .*compressed\)'), 444 re.compile(r'Parse Public EC Key .*compressed\)'), 445 ], 446 # See ecp_light_only 447 'test_suite_ssl': [ 448 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 449 ], 450 } 451 } 452 }, 453 'analyze_driver_vs_reference_ecc_no_bignum': { 454 'test_function': do_analyze_driver_vs_reference, 455 'args': { 456 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', 457 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', 458 'ignored_suites': [ 459 # Modules replaced by drivers 460 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 461 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 462 'bignum.generated', 'bignum.misc', 463 ], 464 'ignored_tests': { 465 'test_suite_config': [ 466 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 467 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 468 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 469 ], 470 'test_suite_platform': [ 471 # Incompatible with sanitizers (e.g. ASan). If the driver 472 # component uses a sanitizer but the reference component 473 # doesn't, we have a PASS vs SKIP mismatch. 474 'Check mbedtls_calloc overallocation', 475 ], 476 # See ecp_light_only 477 'test_suite_random': [ 478 'PSA classic wrapper: ECDSA signature (SECP256R1)', 479 ], 480 # See no_ecp_at_all 481 'test_suite_pkparse': [ 482 re.compile(r'Parse EC Key .*compressed\)'), 483 re.compile(r'Parse Public EC Key .*compressed\)'), 484 ], 485 'test_suite_asn1parse': [ 486 'INTEGER too large for mpi', 487 ], 488 'test_suite_asn1write': [ 489 re.compile(r'ASN.1 Write mpi.*'), 490 ], 491 'test_suite_debug': [ 492 re.compile(r'Debug print mbedtls_mpi.*'), 493 ], 494 # See ecp_light_only 495 'test_suite_ssl': [ 496 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 497 ], 498 } 499 } 500 }, 501 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': { 502 'test_function': do_analyze_driver_vs_reference, 503 'args': { 504 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', 505 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', 506 'ignored_suites': [ 507 # Modules replaced by drivers 508 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', 509 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 510 'bignum.generated', 'bignum.misc', 511 ], 512 'ignored_tests': { 513 'ssl-opt': [ 514 # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C 515 # (because it needs custom groups, which PSA does not 516 # provide), even with MBEDTLS_USE_PSA_CRYPTO. 517 re.compile(r'PSK callback:.*\bdhe-psk\b.*'), 518 ], 519 'test_suite_config': [ 520 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 521 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), 522 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 523 re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'), 524 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 525 ], 526 'test_suite_platform': [ 527 # Incompatible with sanitizers (e.g. ASan). If the driver 528 # component uses a sanitizer but the reference component 529 # doesn't, we have a PASS vs SKIP mismatch. 530 'Check mbedtls_calloc overallocation', 531 ], 532 # See ecp_light_only 533 'test_suite_random': [ 534 'PSA classic wrapper: ECDSA signature (SECP256R1)', 535 ], 536 # See no_ecp_at_all 537 'test_suite_pkparse': [ 538 re.compile(r'Parse EC Key .*compressed\)'), 539 re.compile(r'Parse Public EC Key .*compressed\)'), 540 ], 541 'test_suite_asn1parse': [ 542 'INTEGER too large for mpi', 543 ], 544 'test_suite_asn1write': [ 545 re.compile(r'ASN.1 Write mpi.*'), 546 ], 547 'test_suite_debug': [ 548 re.compile(r'Debug print mbedtls_mpi.*'), 549 ], 550 # See ecp_light_only 551 'test_suite_ssl': [ 552 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', 553 ], 554 } 555 } 556 }, 557 'analyze_driver_vs_reference_ffdh_alg': { 558 'test_function': do_analyze_driver_vs_reference, 559 'args': { 560 'component_ref': 'test_psa_crypto_config_reference_ffdh', 561 'component_driver': 'test_psa_crypto_config_accel_ffdh', 562 'ignored_suites': ['dhm'], 563 'ignored_tests': { 564 'test_suite_config': [ 565 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), 566 ], 567 'test_suite_platform': [ 568 # Incompatible with sanitizers (e.g. ASan). If the driver 569 # component uses a sanitizer but the reference component 570 # doesn't, we have a PASS vs SKIP mismatch. 571 'Check mbedtls_calloc overallocation', 572 ], 573 } 574 } 575 }, 576 'analyze_driver_vs_reference_tfm_config': { 577 'test_function': do_analyze_driver_vs_reference, 578 'args': { 579 'component_ref': 'test_tfm_config', 580 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', 581 'ignored_suites': [ 582 # Modules replaced by drivers 583 'asn1parse', 'asn1write', 584 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 585 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 586 'bignum.generated', 'bignum.misc', 587 ], 588 'ignored_tests': { 589 'test_suite_config': [ 590 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 591 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), 592 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), 593 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') 594 ], 595 'test_suite_config.crypto_combinations': [ 596 'Config: ECC: Weierstrass curves only', 597 ], 598 'test_suite_platform': [ 599 # Incompatible with sanitizers (e.g. ASan). If the driver 600 # component uses a sanitizer but the reference component 601 # doesn't, we have a PASS vs SKIP mismatch. 602 'Check mbedtls_calloc overallocation', 603 ], 604 # See ecp_light_only 605 'test_suite_random': [ 606 'PSA classic wrapper: ECDSA signature (SECP256R1)', 607 ], 608 } 609 } 610 }, 611 'analyze_driver_vs_reference_rsa': { 612 'test_function': do_analyze_driver_vs_reference, 613 'args': { 614 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto', 615 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto', 616 'ignored_suites': [ 617 # Modules replaced by drivers. 618 'rsa', 'pkcs1_v15', 'pkcs1_v21', 619 # We temporarily don't care about PK stuff. 620 'pk', 'pkwrite', 'pkparse' 621 ], 622 'ignored_tests': { 623 'test_suite_config': [ 624 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), 625 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') 626 ], 627 'test_suite_platform': [ 628 # Incompatible with sanitizers (e.g. ASan). If the driver 629 # component uses a sanitizer but the reference component 630 # doesn't, we have a PASS vs SKIP mismatch. 631 'Check mbedtls_calloc overallocation', 632 ], 633 # Following tests depend on RSA_C but are not about 634 # them really, just need to know some error code is there. 635 'test_suite_error': [ 636 'Low and high error', 637 'Single high error' 638 ], 639 # Constant time operations only used for PKCS1_V15 640 'test_suite_constant_time': [ 641 re.compile(r'mbedtls_ct_zeroize_if .*'), 642 re.compile(r'mbedtls_ct_memmove_left .*') 643 ], 644 'test_suite_psa_crypto': [ 645 # We don't support generate_key_custom entry points 646 # in drivers yet. 647 re.compile(r'PSA generate key custom: RSA, e=.*'), 648 re.compile(r'PSA generate key ext: RSA, e=.*'), 649 ], 650 } 651 } 652 }, 653 'analyze_block_cipher_dispatch': { 654 'test_function': do_analyze_driver_vs_reference, 655 'args': { 656 'component_ref': 'test_full_block_cipher_legacy_dispatch', 657 'component_driver': 'test_full_block_cipher_psa_dispatch', 658 'ignored_suites': [ 659 # Skipped in the accelerated component 660 'aes', 'aria', 'camellia', 661 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in 662 # order for the cipher module (actually cipher_wrapper) to work 663 # properly. However these symbols are disabled in the accelerated 664 # component so we ignore them. 665 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', 666 'cipher.camellia', 667 ], 668 'ignored_tests': { 669 'test_suite_config': [ 670 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), 671 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 672 ], 673 'test_suite_cmac': [ 674 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, 675 # but these are not available in the accelerated component. 676 'CMAC null arguments', 677 re.compile('CMAC.* (AES|ARIA|Camellia).*'), 678 ], 679 'test_suite_cipher.padding': [ 680 # Following tests require AES_C/CAMELLIA_C to be enabled, 681 # but these are not available in the accelerated component. 682 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), 683 ], 684 'test_suite_pkcs5': [ 685 # The AES part of PKCS#5 PBES2 is not yet supported. 686 # The rest of PKCS#5 (PBKDF2) works, though. 687 re.compile(r'PBES2 .* AES-.*') 688 ], 689 'test_suite_pkparse': [ 690 # PEM (called by pkparse) requires AES_C in order to decrypt 691 # the key, but this is not available in the accelerated 692 # component. 693 re.compile('Parse RSA Key.*(password|AES-).*'), 694 ], 695 'test_suite_pem': [ 696 # Following tests require AES_C, but this is diabled in the 697 # accelerated component. 698 re.compile('PEM read .*AES.*'), 699 'PEM read (unknown encryption algorithm)', 700 ], 701 'test_suite_error': [ 702 # Following tests depend on AES_C but are not about them 703 # really, just need to know some error code is there. 704 'Single low error', 705 'Low and high error', 706 ], 707 'test_suite_version': [ 708 # Similar to test_suite_error above. 709 'Check for MBEDTLS_AES_C when already present', 710 ], 711 'test_suite_platform': [ 712 # Incompatible with sanitizers (e.g. ASan). If the driver 713 # component uses a sanitizer but the reference component 714 # doesn't, we have a PASS vs SKIP mismatch. 715 'Check mbedtls_calloc overallocation', 716 ], 717 } 718 } 719 } 720} 721 722def main(): 723 main_results = Results() 724 725 try: 726 parser = argparse.ArgumentParser(description=__doc__) 727 parser.add_argument('outcomes', metavar='OUTCOMES.CSV', 728 help='Outcome file to analyze') 729 parser.add_argument('specified_tasks', default='all', nargs='?', 730 help='Analysis to be done. By default, run all tasks. ' 731 'With one or more TASK, run only those. ' 732 'TASK can be the name of a single task or ' 733 'comma/space-separated list of tasks. ') 734 parser.add_argument('--list', action='store_true', 735 help='List all available tasks and exit.') 736 parser.add_argument('--require-full-coverage', action='store_true', 737 dest='full_coverage', help="Require all available " 738 "test cases to be executed and issue an error " 739 "otherwise. This flag is ignored if 'task' is " 740 "neither 'all' nor 'analyze_coverage'") 741 options = parser.parse_args() 742 743 if options.list: 744 for task in KNOWN_TASKS: 745 print(task) 746 sys.exit(0) 747 748 if options.specified_tasks == 'all': 749 tasks_list = KNOWN_TASKS.keys() 750 else: 751 tasks_list = re.split(r'[, ]+', options.specified_tasks) 752 for task in tasks_list: 753 if task not in KNOWN_TASKS: 754 sys.stderr.write('invalid task: {}\n'.format(task)) 755 sys.exit(2) 756 757 KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage 758 759 # If the outcome file exists, parse it once and share the result 760 # among tasks to improve performance. 761 # Otherwise, it will be generated by execute_reference_driver_tests. 762 if not os.path.exists(options.outcomes): 763 if len(tasks_list) > 1: 764 sys.stderr.write("mutiple tasks found, please provide a valid outcomes file.\n") 765 sys.exit(2) 766 767 task_name = tasks_list[0] 768 task = KNOWN_TASKS[task_name] 769 if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable 770 sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name)) 771 sys.exit(2) 772 773 execute_reference_driver_tests(main_results, 774 task['args']['component_ref'], 775 task['args']['component_driver'], 776 options.outcomes) 777 778 outcomes = read_outcome_file(options.outcomes) 779 780 for task in tasks_list: 781 test_function = KNOWN_TASKS[task]['test_function'] 782 test_args = KNOWN_TASKS[task]['args'] 783 test_function(main_results, outcomes, test_args) 784 785 main_results.info("Overall results: {} warnings and {} errors", 786 main_results.warning_count, main_results.error_count) 787 788 sys.exit(0 if (main_results.error_count == 0) else 1) 789 790 except Exception: # pylint: disable=broad-except 791 # Print the backtrace and exit explicitly with our chosen status. 792 traceback.print_exc() 793 sys.exit(120) 794 795if __name__ == '__main__': 796 main() 797
