1#! /usr/bin/env bash 2 3# all.sh 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7 8 9 10################################################################ 11#### Documentation 12################################################################ 13 14# Purpose 15# ------- 16# 17# To run all tests possible or available on the platform. 18# 19# Notes for users 20# --------------- 21# 22# Warning: the test is destructive. It includes various build modes and 23# configurations, and can and will arbitrarily change the current CMake 24# configuration. The following files must be committed into git: 25# * include/mbedtls/mbedtls_config.h 26# * Makefile, library/Makefile, programs/Makefile, tests/Makefile, 27# programs/fuzz/Makefile 28# After running this script, the CMake cache will be lost and CMake 29# will no longer be initialised. 30# 31# The script assumes the presence of a number of tools: 32# * Basic Unix tools (Windows users note: a Unix-style find must be before 33# the Windows find in the PATH) 34# * Perl 35# * GNU Make 36# * CMake 37# * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind) 38# * G++ 39# * arm-gcc and mingw-gcc 40# * ArmCC 5 and ArmCC 6, unless invoked with --no-armcc 41# * OpenSSL and GnuTLS command line tools, in suitable versions for the 42# interoperability tests. The following are the official versions at the 43# time of writing: 44# * GNUTLS_{CLI,SERV} = 3.4.10 45# * GNUTLS_NEXT_{CLI,SERV} = 3.7.2 46# * OPENSSL = 1.0.2g (without Debian/Ubuntu patches) 47# * OPENSSL_NEXT = 3.1.2 48# See the invocation of check_tools below for details. 49# 50# This script must be invoked from the toplevel directory of a git 51# working copy of Mbed TLS. 52# 53# The behavior on an error depends on whether --keep-going (alias -k) 54# is in effect. 55# * Without --keep-going: the script stops on the first error without 56# cleaning up. This lets you work in the configuration of the failing 57# component. 58# * With --keep-going: the script runs all requested components and 59# reports failures at the end. In particular the script always cleans 60# up on exit. 61# 62# Note that the output is not saved. You may want to run 63# script -c tests/scripts/all.sh 64# or 65# tests/scripts/all.sh >all.log 2>&1 66# 67# Notes for maintainers 68# --------------------- 69# 70# The bulk of the code is organized into functions that follow one of the 71# following naming conventions: 72# * pre_XXX: things to do before running the tests, in order. 73# * component_XXX: independent components. They can be run in any order. 74# * component_check_XXX: quick tests that aren't worth parallelizing. 75# * component_build_XXX: build things but don't run them. 76# * component_test_XXX: build and test. 77# * component_release_XXX: tests that the CI should skip during PR testing. 78# * support_XXX: if support_XXX exists and returns false then 79# component_XXX is not run by default. 80# * post_XXX: things to do after running the tests. 81# * other: miscellaneous support functions. 82# 83# Each component must start by invoking `msg` with a short informative message. 84# 85# Warning: due to the way bash detects errors, the failure of a command 86# inside 'if' or '!' is not detected. Use the 'not' function instead of '!'. 87# 88# Each component is executed in a separate shell process. The component 89# fails if any command in it returns a non-zero status. 90# 91# The framework performs some cleanup tasks after each component. This 92# means that components can assume that the working directory is in a 93# cleaned-up state, and don't need to perform the cleanup themselves. 94# * Run `make clean`. 95# * Restore `include/mbedtls/mbedtls_config.h` from a backup made before running 96# the component. 97# * Check out `Makefile`, `library/Makefile`, `programs/Makefile`, 98# `tests/Makefile` and `programs/fuzz/Makefile` from git. 99# This cleans up after an in-tree use of CMake. 100# 101# The tests are roughly in order from fastest to slowest. This doesn't 102# have to be exact, but in general you should add slower tests towards 103# the end and fast checks near the beginning. 104 105 106 107################################################################ 108#### Initialization and command line parsing 109################################################################ 110 111# Abort on errors (even on the left-hand side of a pipe). 112# Treat uninitialised variables as errors. 113set -e -o pipefail -u 114 115# Enable ksh/bash extended file matching patterns 116shopt -s extglob 117 118# For project detection 119in_mbedtls_repo () { 120 test "$PROJECT_NAME" = "Mbed TLS" 121} 122 123in_tf_psa_crypto_repo () { 124 test "$PROJECT_NAME" = "TF-PSA-Crypto" 125} 126 127pre_check_environment () { 128 # For project detection 129 PROJECT_NAME_FILE='./scripts/project_name.txt' 130 if read -r PROJECT_NAME < "$PROJECT_NAME_FILE"; then :; else 131 echo "$PROJECT_NAME_FILE does not exist... Exiting..." >&2 132 exit 1 133 fi 134 135 if in_mbedtls_repo || in_tf_psa_crypto_repo; then :; else 136 echo "Must be run from Mbed TLS / TF-PSA-Crypto root" >&2 137 exit 1 138 fi 139} 140 141pre_initialize_variables () { 142 if in_mbedtls_repo; then 143 CONFIG_H='include/mbedtls/mbedtls_config.h' 144 else 145 CONFIG_H='drivers/builtin/include/mbedtls/mbedtls_config.h' 146 fi 147 CRYPTO_CONFIG_H='include/psa/crypto_config.h' 148 CONFIG_TEST_DRIVER_H='tests/include/test/drivers/config_test_driver.h' 149 150 # Files that are clobbered by some jobs will be backed up. Use a different 151 # suffix from auxiliary scripts so that all.sh and auxiliary scripts can 152 # independently decide when to remove the backup file. 153 backup_suffix='.all.bak' 154 # Files clobbered by config.py 155 files_to_back_up="$CONFIG_H $CRYPTO_CONFIG_H $CONFIG_TEST_DRIVER_H" 156 if in_mbedtls_repo; then 157 # Files clobbered by in-tree cmake 158 files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile" 159 fi 160 161 append_outcome=0 162 MEMORY=0 163 FORCE=0 164 QUIET=0 165 KEEP_GOING=0 166 167 # Seed value used with the --release-test option. 168 # 169 # See also RELEASE_SEED in basic-build-test.sh. Debugging is easier if 170 # both values are kept in sync. If you change the value here because it 171 # breaks some tests, you'll definitely want to change it in 172 # basic-build-test.sh as well. 173 RELEASE_SEED=1 174 175 # Specify character collation for regular expressions and sorting with C locale 176 export LC_COLLATE=C 177 178 : ${MBEDTLS_TEST_OUTCOME_FILE=} 179 : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} 180 export MBEDTLS_TEST_OUTCOME_FILE 181 export MBEDTLS_TEST_PLATFORM 182 183 # Default commands, can be overridden by the environment 184 : ${OPENSSL:="openssl"} 185 : ${OPENSSL_NEXT:="$OPENSSL"} 186 : ${GNUTLS_CLI:="gnutls-cli"} 187 : ${GNUTLS_SERV:="gnutls-serv"} 188 : ${OUT_OF_SOURCE_DIR:=./mbedtls_out_of_source_build} 189 : ${ARMC5_BIN_DIR:=/usr/bin} 190 : ${ARMC6_BIN_DIR:=/usr/bin} 191 : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} 192 : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} 193 : ${CLANG_LATEST:="clang-latest"} 194 : ${CLANG_EARLIEST:="clang-earliest"} 195 : ${GCC_LATEST:="gcc-latest"} 196 : ${GCC_EARLIEST:="gcc-earliest"} 197 # if MAKEFLAGS is not set add the -j option to speed up invocations of make 198 if [ -z "${MAKEFLAGS+set}" ]; then 199 export MAKEFLAGS="-j$(all_sh_nproc)" 200 fi 201 # if CC is not set, use clang by default (if present) to improve build times 202 if [ -z "${CC+set}" ] && (type clang > /dev/null 2>&1); then 203 export CC="clang" 204 fi 205 206 if [ -n "${OPENSSL_3+set}" ]; then 207 export OPENSSL_NEXT="$OPENSSL_3" 208 fi 209 210 # Include more verbose output for failing tests run by CMake or make 211 export CTEST_OUTPUT_ON_FAILURE=1 212 213 # CFLAGS and LDFLAGS for Asan builds that don't use CMake 214 # default to -O2, use -Ox _after_ this if you want another level 215 ASAN_CFLAGS='-O2 -Werror -fsanitize=address,undefined -fno-sanitize-recover=all' 216 # Normally, tests should use this compiler for ASAN testing 217 ASAN_CC=clang 218 219 # Platform tests have an allocation that returns null 220 export ASAN_OPTIONS="allocator_may_return_null=1" 221 export MSAN_OPTIONS="allocator_may_return_null=1" 222 223 # Gather the list of available components. These are the functions 224 # defined in this script whose name starts with "component_". 225 ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//') 226 227 # Delay determining SUPPORTED_COMPONENTS until the command line options have a chance to override 228 # the commands set by the environment 229} 230 231setup_quiet_wrappers() 232{ 233 # Pick up "quiet" wrappers for make and cmake, which don't output very much 234 # unless there is an error. This reduces logging overhead in the CI. 235 # 236 # Note that the cmake wrapper breaks unless we use an absolute path here. 237 if [[ -e ${PWD}/tests/scripts/quiet ]]; then 238 export PATH=${PWD}/tests/scripts/quiet:$PATH 239 fi 240} 241 242# Test whether the component $1 is included in the command line patterns. 243is_component_included() 244{ 245 # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS 246 # only does word splitting. 247 set -f 248 for pattern in $COMMAND_LINE_COMPONENTS; do 249 set +f 250 case ${1#component_} in $pattern) return 0;; esac 251 done 252 set +f 253 return 1 254} 255 256usage() 257{ 258 cat <<EOF 259Usage: $0 [OPTION]... [COMPONENT]... 260Run mbedtls release validation tests. 261By default, run all tests. With one or more COMPONENT, run only those. 262COMPONENT can be the name of a component or a shell wildcard pattern. 263 264Examples: 265 $0 "check_*" 266 Run all sanity checks. 267 $0 --no-armcc --except test_memsan 268 Run everything except builds that require armcc and MemSan. 269 270Special options: 271 -h|--help Print this help and exit. 272 --list-all-components List all available test components and exit. 273 --list-components List components supported on this platform and exit. 274 275General options: 276 -q|--quiet Only output component names, and errors if any. 277 -f|--force Force the tests to overwrite any modified files. 278 -k|--keep-going Run all tests and report errors at the end. 279 -m|--memory Additional optional memory tests. 280 --append-outcome Append to the outcome file (if used). 281 --arm-none-eabi-gcc-prefix=<string> 282 Prefix for a cross-compiler for arm-none-eabi 283 (default: "${ARM_NONE_EABI_GCC_PREFIX}") 284 --arm-linux-gnueabi-gcc-prefix=<string> 285 Prefix for a cross-compiler for arm-linux-gnueabi 286 (default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}") 287 --armcc Run ARM Compiler builds (on by default). 288 --restore First clean up the build tree, restoring backed up 289 files. Do not run any components unless they are 290 explicitly specified. 291 --error-test Error test mode: run a failing function in addition 292 to any specified component. May be repeated. 293 --except Exclude the COMPONENTs listed on the command line, 294 instead of running only those. 295 --no-append-outcome Write a new outcome file and analyze it (default). 296 --no-armcc Skip ARM Compiler builds. 297 --no-force Refuse to overwrite modified files (default). 298 --no-keep-going Stop at the first error (default). 299 --no-memory No additional memory tests (default). 300 --no-quiet Print full output from components. 301 --out-of-source-dir=<path> Directory used for CMake out-of-source build tests. 302 --outcome-file=<path> File where test outcomes are written (not done if 303 empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). 304 --random-seed Use a random seed value for randomized tests (default). 305 -r|--release-test Run this script in release mode. This fixes the seed value to ${RELEASE_SEED}. 306 -s|--seed Integer seed value to use for this test run. 307 308Tool path options: 309 --armc5-bin-dir=<ARMC5_bin_dir_path> ARM Compiler 5 bin directory. 310 --armc6-bin-dir=<ARMC6_bin_dir_path> ARM Compiler 6 bin directory. 311 --clang-earliest=<Clang_earliest_path> Earliest version of clang available 312 --clang-latest=<Clang_latest_path> Latest version of clang available 313 --gcc-earliest=<GCC_earliest_path> Earliest version of GCC available 314 --gcc-latest=<GCC_latest_path> Latest version of GCC available 315 --gnutls-cli=<GnuTLS_cli_path> GnuTLS client executable to use for most tests. 316 --gnutls-serv=<GnuTLS_serv_path> GnuTLS server executable to use for most tests. 317 --openssl=<OpenSSL_path> OpenSSL executable to use for most tests. 318 --openssl-next=<OpenSSL_path> OpenSSL executable to use for recent things like ARIA 319EOF 320} 321 322# Cleanup before/after running a component. 323# Remove built files as well as the cmake cache/config. 324# Does not remove generated source files. 325cleanup() 326{ 327 if in_mbedtls_repo; then 328 command make clean 329 fi 330 331 # Remove CMake artefacts 332 find . -name .git -prune -o \ 333 -iname CMakeFiles -exec rm -rf {} \+ -o \ 334 \( -iname cmake_install.cmake -o \ 335 -iname CTestTestfile.cmake -o \ 336 -iname CMakeCache.txt -o \ 337 -path './cmake/*.cmake' \) -exec rm -f {} \+ 338 # Remove Makefiles generated by in-tree CMake builds 339 rm -f 3rdparty/Makefile 3rdparty/*/Makefile pkgconfig/Makefile framework/Makefile 340 rm -f include/Makefile programs/!(fuzz)/Makefile 341 342 # Remove any artifacts from the component_test_cmake_as_subdirectory test. 343 rm -rf programs/test/cmake_subproject/build 344 rm -f programs/test/cmake_subproject/Makefile 345 rm -f programs/test/cmake_subproject/cmake_subproject 346 347 # Remove any artifacts from the component_test_cmake_as_package test. 348 rm -rf programs/test/cmake_package/build 349 rm -f programs/test/cmake_package/Makefile 350 rm -f programs/test/cmake_package/cmake_package 351 352 # Remove any artifacts from the component_test_cmake_as_installed_package test. 353 rm -rf programs/test/cmake_package_install/build 354 rm -f programs/test/cmake_package_install/Makefile 355 rm -f programs/test/cmake_package_install/cmake_package_install 356 357 # Restore files that may have been clobbered by the job 358 for x in $files_to_back_up; do 359 if [[ -e "$x$backup_suffix" ]]; then 360 cp -p "$x$backup_suffix" "$x" 361 fi 362 done 363} 364 365# Final cleanup when this script exits (except when exiting on a failure 366# in non-keep-going mode). 367final_cleanup () { 368 cleanup 369 370 for x in $files_to_back_up; do 371 rm -f "$x$backup_suffix" 372 done 373} 374 375# Executed on exit. May be redefined depending on command line options. 376final_report () { 377 : 378} 379 380fatal_signal () { 381 final_cleanup 382 final_report $1 383 trap - $1 384 kill -$1 $$ 385} 386 387trap 'fatal_signal HUP' HUP 388trap 'fatal_signal INT' INT 389trap 'fatal_signal TERM' TERM 390 391# Number of processors on this machine. Used as the default setting 392# for parallel make. 393all_sh_nproc () 394{ 395 { 396 nproc || # Linux 397 sysctl -n hw.ncpuonline || # NetBSD, OpenBSD 398 sysctl -n hw.ncpu || # FreeBSD 399 echo 1 400 } 2>/dev/null 401} 402 403msg() 404{ 405 if [ -n "${current_component:-}" ]; then 406 current_section="${current_component#component_}: $1" 407 else 408 current_section="$1" 409 fi 410 411 if [ $QUIET -eq 1 ]; then 412 return 413 fi 414 415 echo "" 416 echo "******************************************************************" 417 echo "* $current_section " 418 printf "* "; date 419 echo "******************************************************************" 420} 421 422armc6_build_test() 423{ 424 FLAGS="$1" 425 426 msg "build: ARM Compiler 6 ($FLAGS)" 427 make clean 428 ARM_TOOL_VARIANT="ult" CC="$ARMC6_CC" AR="$ARMC6_AR" CFLAGS="$FLAGS" \ 429 WARNING_CFLAGS='-Werror -xc -std=c99' make lib 430 431 msg "size: ARM Compiler 6 ($FLAGS)" 432 "$ARMC6_FROMELF" -z library/*.o 433} 434 435err_msg() 436{ 437 echo "$1" >&2 438} 439 440check_tools() 441{ 442 for tool in "$@"; do 443 if ! `type "$tool" >/dev/null 2>&1`; then 444 err_msg "$tool not found!" 445 exit 1 446 fi 447 done 448} 449 450pre_parse_command_line () { 451 COMMAND_LINE_COMPONENTS= 452 all_except=0 453 error_test=0 454 list_components=0 455 restore_first=0 456 no_armcc= 457 458 # Note that legacy options are ignored instead of being omitted from this 459 # list of options, so invocations that worked with previous version of 460 # all.sh will still run and work properly. 461 while [ $# -gt 0 ]; do 462 case "$1" in 463 --append-outcome) append_outcome=1;; 464 --arm-none-eabi-gcc-prefix) shift; ARM_NONE_EABI_GCC_PREFIX="$1";; 465 --arm-linux-gnueabi-gcc-prefix) shift; ARM_LINUX_GNUEABI_GCC_PREFIX="$1";; 466 --armcc) no_armcc=;; 467 --armc5-bin-dir) shift; ARMC5_BIN_DIR="$1";; 468 --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; 469 --clang-earliest) shift; CLANG_EARLIEST="$1";; 470 --clang-latest) shift; CLANG_LATEST="$1";; 471 --error-test) error_test=$((error_test + 1));; 472 --except) all_except=1;; 473 --force|-f) FORCE=1;; 474 --gcc-earliest) shift; GCC_EARLIEST="$1";; 475 --gcc-latest) shift; GCC_LATEST="$1";; 476 --gnutls-cli) shift; GNUTLS_CLI="$1";; 477 --gnutls-legacy-cli) shift;; # ignored for backward compatibility 478 --gnutls-legacy-serv) shift;; # ignored for backward compatibility 479 --gnutls-serv) shift; GNUTLS_SERV="$1";; 480 --help|-h) usage; exit;; 481 --keep-going|-k) KEEP_GOING=1;; 482 --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; 483 --list-components) list_components=1;; 484 --memory|-m) MEMORY=1;; 485 --no-append-outcome) append_outcome=0;; 486 --no-armcc) no_armcc=1;; 487 --no-force) FORCE=0;; 488 --no-keep-going) KEEP_GOING=0;; 489 --no-memory) MEMORY=0;; 490 --no-quiet) QUIET=0;; 491 --openssl) shift; OPENSSL="$1";; 492 --openssl-next) shift; OPENSSL_NEXT="$1";; 493 --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; 494 --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; 495 --quiet|-q) QUIET=1;; 496 --random-seed) unset SEED;; 497 --release-test|-r) SEED=$RELEASE_SEED;; 498 --restore) restore_first=1;; 499 --seed|-s) shift; SEED="$1";; 500 -*) 501 echo >&2 "Unknown option: $1" 502 echo >&2 "Run $0 --help for usage." 503 exit 120 504 ;; 505 *) COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS $1";; 506 esac 507 shift 508 done 509 510 # Exclude components that are not supported on this platform. 511 SUPPORTED_COMPONENTS= 512 for component in $ALL_COMPONENTS; do 513 case $(type "support_$component" 2>&1) in 514 *' function'*) 515 if ! support_$component; then continue; fi;; 516 esac 517 SUPPORTED_COMPONENTS="$SUPPORTED_COMPONENTS $component" 518 done 519 520 if [ $list_components -eq 1 ]; then 521 printf '%s\n' $SUPPORTED_COMPONENTS 522 exit 523 fi 524 525 # With no list of components, run everything. 526 if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then 527 all_except=1 528 fi 529 530 # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. 531 # Ignore it if components are listed explicitly on the command line. 532 if [ -n "$no_armcc" ] && [ $all_except -eq 1 ]; then 533 COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" 534 fi 535 536 # Error out if an explicitly requested component doesn't exist. 537 if [ $all_except -eq 0 ]; then 538 unsupported=0 539 # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS 540 # only does word splitting. 541 set -f 542 for component in $COMMAND_LINE_COMPONENTS; do 543 set +f 544 # If the requested name includes a wildcard character, don't 545 # check it. Accept wildcard patterns that don't match anything. 546 case $component in 547 *[*?\[]*) continue;; 548 esac 549 case " $SUPPORTED_COMPONENTS " in 550 *" $component "*) :;; 551 *) 552 echo >&2 "Component $component was explicitly requested, but is not known or not supported." 553 unsupported=$((unsupported + 1));; 554 esac 555 done 556 set +f 557 if [ $unsupported -ne 0 ]; then 558 exit 2 559 fi 560 fi 561 562 # Build the list of components to run. 563 RUN_COMPONENTS= 564 for component in $SUPPORTED_COMPONENTS; do 565 if is_component_included "$component"; [ $? -eq $all_except ]; then 566 RUN_COMPONENTS="$RUN_COMPONENTS $component" 567 fi 568 done 569 570 unset all_except 571 unset no_armcc 572} 573 574pre_check_git () { 575 if [ $FORCE -eq 1 ]; then 576 rm -rf "$OUT_OF_SOURCE_DIR" 577 git checkout-index -f -q $CONFIG_H 578 cleanup 579 else 580 581 if [ -d "$OUT_OF_SOURCE_DIR" ]; then 582 echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 583 echo "You can either delete this directory manually, or force the test by rerunning" 584 echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" 585 exit 1 586 fi 587 588 if ! git diff --quiet "$CONFIG_H"; then 589 err_msg "Warning - the configuration file '$CONFIG_H' has been edited. " 590 echo "You can either delete or preserve your work, or force the test by rerunning the" 591 echo "script as: $0 --force" 592 exit 1 593 fi 594 fi 595} 596 597pre_restore_files () { 598 # If the makefiles have been generated by a framework such as cmake, 599 # restore them from git. If the makefiles look like modifications from 600 # the ones checked into git, take care not to modify them. Whatever 601 # this function leaves behind is what the script will restore before 602 # each component. 603 case "$(head -n1 Makefile)" in 604 *[Gg]enerated*) 605 git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile 606 git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile 607 ;; 608 esac 609} 610 611pre_back_up () { 612 for x in $files_to_back_up; do 613 cp -p "$x" "$x$backup_suffix" 614 done 615} 616 617pre_setup_keep_going () { 618 failure_count=0 # Number of failed components 619 last_failure_status=0 # Last failure status in this component 620 621 # See err_trap 622 previous_failure_status=0 623 previous_failed_command= 624 previous_failure_funcall_depth=0 625 unset report_failed_command 626 627 start_red= 628 end_color= 629 if [ -t 1 ]; then 630 case "${TERM:-}" in 631 *color*|cygwin|linux|rxvt*|screen|[Eex]term*) 632 start_red=$(printf '\033[31m') 633 end_color=$(printf '\033[0m') 634 ;; 635 esac 636 fi 637 638 # Keep a summary of failures in a file. We'll print it out at the end. 639 failure_summary_file=$PWD/all-sh-failures-$$.log 640 : >"$failure_summary_file" 641 642 # Whether it makes sense to keep a component going after the specified 643 # command fails (test command) or not (configure or build). 644 # This function normally receives the failing simple command 645 # ($BASH_COMMAND) as an argument, but if $report_failed_command is set, 646 # this is passed instead. 647 # This doesn't have to be 100% accurate: all failures are recorded anyway. 648 # False positives result in running things that can't be expected to 649 # work. False negatives result in things not running after something else 650 # failed even though they might have given useful feedback. 651 can_keep_going_after_failure () { 652 case "$1" in 653 "msg "*) false;; 654 "cd "*) false;; 655 "diff "*) true;; 656 *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ... 657 *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ... 658 *make*check*) true;; 659 "grep "*) true;; 660 "[ "*) true;; 661 "! "*) true;; 662 *) false;; 663 esac 664 } 665 666 # This function runs if there is any error in a component. 667 # It must either exit with a nonzero status, or set 668 # last_failure_status to a nonzero value. 669 err_trap () { 670 # Save $? (status of the failing command). This must be the very 671 # first thing, before $? is overridden. 672 last_failure_status=$? 673 failed_command=${report_failed_command-$BASH_COMMAND} 674 675 if [[ $last_failure_status -eq $previous_failure_status && 676 "$failed_command" == "$previous_failed_command" && 677 ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]] 678 then 679 # The same command failed twice in a row, but this time one level 680 # less deep in the function call stack. This happens when the last 681 # command of a function returns a nonzero status, and the function 682 # returns that same status. Ignore the second failure. 683 previous_failure_funcall_depth=${#FUNCNAME[@]} 684 return 685 fi 686 previous_failure_status=$last_failure_status 687 previous_failed_command=$failed_command 688 previous_failure_funcall_depth=${#FUNCNAME[@]} 689 690 text="$current_section: $failed_command -> $last_failure_status" 691 echo "${start_red}^^^^$text^^^^${end_color}" >&2 692 echo "$text" >>"$failure_summary_file" 693 694 # If the command is fatal (configure or build command), stop this 695 # component. Otherwise (test command) keep the component running 696 # (run more tests from the same build). 697 if ! can_keep_going_after_failure "$failed_command"; then 698 exit $last_failure_status 699 fi 700 } 701 702 final_report () { 703 if [ $failure_count -gt 0 ]; then 704 echo 705 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" 706 echo "${start_red}FAILED: $failure_count components${end_color}" 707 cat "$failure_summary_file" 708 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" 709 elif [ -z "${1-}" ]; then 710 echo "SUCCESS :)" 711 fi 712 if [ -n "${1-}" ]; then 713 echo "Killed by SIG$1." 714 fi 715 rm -f "$failure_summary_file" 716 if [ $failure_count -gt 0 ]; then 717 exit 1 718 fi 719 } 720} 721 722# record_status() and if_build_succeeded() are kept temporarily for backward 723# compatibility. Don't use them in new components. 724record_status () { 725 "$@" 726} 727if_build_succeeded () { 728 "$@" 729} 730 731# '! true' does not trigger the ERR trap. Arrange to trigger it, with 732# a reasonably informative error message (not just "$@"). 733not () { 734 if "$@"; then 735 report_failed_command="! $*" 736 false 737 unset report_failed_command 738 fi 739} 740 741pre_prepare_outcome_file () { 742 case "$MBEDTLS_TEST_OUTCOME_FILE" in 743 [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; 744 esac 745 if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then 746 rm -f "$MBEDTLS_TEST_OUTCOME_FILE" 747 fi 748} 749 750pre_print_configuration () { 751 if [ $QUIET -eq 1 ]; then 752 return 753 fi 754 755 msg "info: $0 configuration" 756 echo "MEMORY: $MEMORY" 757 echo "FORCE: $FORCE" 758 echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" 759 echo "SEED: ${SEED-"UNSET"}" 760 echo 761 echo "OPENSSL: $OPENSSL" 762 echo "OPENSSL_NEXT: $OPENSSL_NEXT" 763 echo "GNUTLS_CLI: $GNUTLS_CLI" 764 echo "GNUTLS_SERV: $GNUTLS_SERV" 765 echo "ARMC5_BIN_DIR: $ARMC5_BIN_DIR" 766 echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" 767} 768 769# Make sure the tools we need are available. 770pre_check_tools () { 771 # Build the list of variables to pass to output_env.sh. 772 set env 773 774 case " $RUN_COMPONENTS " in 775 # Require OpenSSL and GnuTLS if running any tests (as opposed to 776 # only doing builds). Not all tests run OpenSSL and GnuTLS, but this 777 # is a good enough approximation in practice. 778 *" test_"* | *" release_test_"*) 779 # To avoid setting OpenSSL and GnuTLS for each call to compat.sh 780 # and ssl-opt.sh, we just export the variables they require. 781 export OPENSSL="$OPENSSL" 782 export GNUTLS_CLI="$GNUTLS_CLI" 783 export GNUTLS_SERV="$GNUTLS_SERV" 784 # Avoid passing --seed flag in every call to ssl-opt.sh 785 if [ -n "${SEED-}" ]; then 786 export SEED 787 fi 788 set "$@" OPENSSL="$OPENSSL" 789 set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" 790 check_tools "$OPENSSL" "$OPENSSL_NEXT" \ 791 "$GNUTLS_CLI" "$GNUTLS_SERV" 792 ;; 793 esac 794 795 case " $RUN_COMPONENTS " in 796 *_doxygen[_\ ]*) check_tools "doxygen" "dot";; 797 esac 798 799 case " $RUN_COMPONENTS " in 800 *_arm_none_eabi_gcc[_\ ]*) check_tools "${ARM_NONE_EABI_GCC_PREFIX}gcc";; 801 esac 802 803 case " $RUN_COMPONENTS " in 804 *_mingw[_\ ]*) check_tools "i686-w64-mingw32-gcc";; 805 esac 806 807 case " $RUN_COMPONENTS " in 808 *" test_zeroize "*) check_tools "gdb";; 809 esac 810 811 case " $RUN_COMPONENTS " in 812 *_armcc*) 813 ARMC5_CC="$ARMC5_BIN_DIR/armcc" 814 ARMC5_AR="$ARMC5_BIN_DIR/armar" 815 ARMC5_FROMELF="$ARMC5_BIN_DIR/fromelf" 816 ARMC6_CC="$ARMC6_BIN_DIR/armclang" 817 ARMC6_AR="$ARMC6_BIN_DIR/armar" 818 ARMC6_FROMELF="$ARMC6_BIN_DIR/fromelf" 819 check_tools "$ARMC5_CC" "$ARMC5_AR" "$ARMC5_FROMELF" \ 820 "$ARMC6_CC" "$ARMC6_AR" "$ARMC6_FROMELF";; 821 esac 822 823 # past this point, no call to check_tool, only printing output 824 if [ $QUIET -eq 1 ]; then 825 return 826 fi 827 828 msg "info: output_env.sh" 829 case $RUN_COMPONENTS in 830 *_armcc*) 831 set "$@" ARMC5_CC="$ARMC5_CC" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; 832 *) set "$@" RUN_ARMCC=0;; 833 esac 834 "$@" scripts/output_env.sh 835} 836 837pre_generate_files () { 838 # since make doesn't have proper dependencies, remove any possibly outdate 839 # file that might be around before generating fresh ones 840 make neat 841 if [ $QUIET -eq 1 ]; then 842 make generated_files >/dev/null 843 else 844 make generated_files 845 fi 846} 847 848clang_version () { 849 if command -v clang > /dev/null ; then 850 clang --version|grep version|sed -E 's#.*version ([0-9]+).*#\1#' 851 else 852 echo 0 # report version 0 for "no clang" 853 fi 854} 855 856################################################################ 857#### Helpers for components using libtestdriver1 858################################################################ 859 860# How to use libtestdriver1 861# ------------------------- 862# 863# 1. Define the list algorithms and key types to accelerate, 864# designated the same way as PSA_WANT_ macros but without PSA_WANT_. 865# Examples: 866# - loc_accel_list="ALG_JPAKE" 867# - loc_accel_list="ALG_FFDH KEY_TYPE_DH_KEY_PAIR KEY_TYPE_DH_PUBLIC_KEY" 868# 2. Make configurations changes for the driver and/or main libraries. 869# 2a. Call helper_libtestdriver1_adjust_config <base>, where the argument 870# can be either "default" to start with the default config, or a name 871# supported by scripts/config.py (for example, "full"). This selects 872# the base to use, and makes common adjustments. 873# 2b. If desired, adjust the PSA_WANT symbols in psa/crypto_config.h. 874# These changes affect both the driver and the main libraries. 875# (Note: they need to have the same set of PSA_WANT symbols, as that 876# determines the ABI between them.) 877# 2c. Adjust MBEDTLS_ symbols in mbedtls_config.h. This only affects the 878# main libraries. Typically, you want to disable the module(s) that are 879# being accelerated. You may need to also disable modules that depend 880# on them or options that are not supported with drivers. 881# 2d. On top of psa/crypto_config.h, the driver library uses its own config 882# file: tests/include/test/drivers/config_test_driver.h. You usually 883# don't need to edit it: using loc_extra_list (see below) is preferred. 884# However, when there's no PSA symbol for what you want to enable, 885# calling scripts/config.py on this file remains the only option. 886# 3. Build the driver library, then the main libraries, test, and programs. 887# 3a. Call helper_libtestdriver1_make_drivers "$loc_accel_list". You may 888# need to enable more algorithms here, typically hash algorithms when 889# accelerating some signature algorithms (ECDSA, RSAv2). This is done 890# by passing a 2nd argument listing the extra algorithms. 891# Example: 892# loc_extra_list="ALG_SHA_224 ALG_SHA_256 ALG_SHA_384 ALG_SHA_512" 893# helper_libtestdriver1_make_drivers "$loc_accel_list" "$loc_extra_list" 894# 3b. Call helper_libtestdriver1_make_main "$loc_accel_list". Any 895# additional arguments will be passed to make: this can be useful if 896# you don't want to build everything when iterating during development. 897# Example: 898# helper_libtestdriver1_make_main "$loc_accel_list" -C tests test_suite_foo 899# 4. Run the tests you want. 900 901# Adjust the configuration - for both libtestdriver1 and main library, 902# as they should have the same PSA_WANT macros. 903helper_libtestdriver1_adjust_config () { 904 base_config=$1 905 # Select the base configuration 906 if [ "$base_config" != "default" ]; then 907 scripts/config.py "$base_config" 908 fi 909 910 # Enable PSA-based config (necessary to use drivers) 911 scripts/config.py set MBEDTLS_PSA_CRYPTO_CONFIG 912 913 # Dynamic secure element support is a deprecated feature and needs to be disabled here. 914 # This is done to have the same form of psa_key_attributes_s for libdriver and library. 915 scripts/config.py unset MBEDTLS_PSA_CRYPTO_SE_C 916 917 # If threading is enabled on the normal build, then we need to enable it in the drivers as well, 918 # otherwise we will end up running multithreaded tests without mutexes to protect them. 919 if scripts/config.py get MBEDTLS_THREADING_C; then 920 scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_C 921 fi 922 923 if scripts/config.py get MBEDTLS_THREADING_PTHREAD; then 924 scripts/config.py -f "$CONFIG_TEST_DRIVER_H" set MBEDTLS_THREADING_PTHREAD 925 fi 926} 927 928# When called with no parameter this function disables all builtin curves. 929# The function optionally accepts 1 parameter: a space-separated list of the 930# curves that should be kept enabled. 931helper_disable_builtin_curves () { 932 allowed_list="${1:-}" 933 scripts/config.py unset-all "MBEDTLS_ECP_DP_[0-9A-Z_a-z]*_ENABLED" 934 935 for curve in $allowed_list; do 936 scripts/config.py set $curve 937 done 938} 939 940# Helper returning the list of supported elliptic curves from CRYPTO_CONFIG_H, 941# without the "PSA_WANT_" prefix. This becomes handy for accelerating curves 942# in the following helpers. 943helper_get_psa_curve_list () { 944 loc_list="" 945 for item in $(sed -n 's/^#define PSA_WANT_\(ECC_[0-9A-Z_a-z]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do 946 loc_list="$loc_list $item" 947 done 948 949 echo "$loc_list" 950} 951 952# Helper returning the list of supported DH groups from CRYPTO_CONFIG_H, 953# without the "PSA_WANT_" prefix. This becomes handy for accelerating DH groups 954# in the following helpers. 955helper_get_psa_dh_group_list () { 956 loc_list="" 957 for item in $(sed -n 's/^#define PSA_WANT_\(DH_RFC7919_[0-9]*\).*/\1/p' <"$CRYPTO_CONFIG_H"); do 958 loc_list="$loc_list $item" 959 done 960 961 echo "$loc_list" 962} 963 964# Get the list of uncommented PSA_WANT_KEY_TYPE_xxx_ from CRYPTO_CONFIG_H. This 965# is useful to easily get a list of key type symbols to accelerate. 966# The function accepts a single argument which is the key type: ECC, DH, RSA. 967helper_get_psa_key_type_list () { 968 key_type="$1" 969 loc_list="" 970 for item in $(sed -n "s/^#define PSA_WANT_\(KEY_TYPE_${key_type}_[0-9A-Z_a-z]*\).*/\1/p" <"$CRYPTO_CONFIG_H"); do 971 # Skip DERIVE for elliptic keys since there is no driver dispatch for 972 # it so it cannot be accelerated. 973 if [ "$item" != "KEY_TYPE_ECC_KEY_PAIR_DERIVE" ]; then 974 loc_list="$loc_list $item" 975 fi 976 done 977 978 echo "$loc_list" 979} 980 981# Build the drivers library libtestdriver1.a (with ASan). 982# 983# Parameters: 984# 1. a space-separated list of things to accelerate; 985# 2. optional: a space-separate list of things to also support. 986# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. 987helper_libtestdriver1_make_drivers () { 988 loc_accel_flags=$( echo "$1 ${2-}" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) 989 make CC=$ASAN_CC -C tests libtestdriver1.a CFLAGS=" $ASAN_CFLAGS $loc_accel_flags" LDFLAGS="$ASAN_CFLAGS" 990} 991 992# Build the main libraries, programs and tests, 993# linking to the drivers library (with ASan). 994# 995# Parameters: 996# 1. a space-separated list of things to accelerate; 997# *. remaining arguments if any are passed directly to make 998# (examples: lib, -C tests test_suite_xxx, etc.) 999# Here "things" are PSA_WANT_ symbols but with PSA_WANT_ removed. 1000helper_libtestdriver1_make_main () { 1001 loc_accel_list=$1 1002 shift 1003 1004 # we need flags both with and without the LIBTESTDRIVER1_ prefix 1005 loc_accel_flags=$( echo "$loc_accel_list" | sed 's/[^ ]* */-DLIBTESTDRIVER1_MBEDTLS_PSA_ACCEL_&/g' ) 1006 loc_accel_flags="$loc_accel_flags $( echo "$loc_accel_list" | sed 's/[^ ]* */-DMBEDTLS_PSA_ACCEL_&/g' )" 1007 make CC=$ASAN_CC CFLAGS="$ASAN_CFLAGS -I../tests/include -I../tests -I../../tests -DPSA_CRYPTO_DRIVER_TEST -DMBEDTLS_TEST_LIBTESTDRIVER1 $loc_accel_flags" LDFLAGS="-ltestdriver1 $ASAN_CFLAGS" "$@" 1008} 1009 1010# Include the components from components.sh 1011test_script_dir="${0%/*}" 1012for file in "$test_script_dir"/components*.sh; do 1013 source $file 1014done 1015 1016################################################################ 1017#### Termination 1018################################################################ 1019 1020post_report () { 1021 msg "Done, cleaning up" 1022 final_cleanup 1023 1024 final_report 1025} 1026 1027################################################################ 1028#### Run all the things 1029################################################################ 1030 1031# Function invoked by --error-test to test error reporting. 1032pseudo_component_error_test () { 1033 msg "Testing error reporting $error_test_i" 1034 if [ $KEEP_GOING -ne 0 ]; then 1035 echo "Expect three failing commands." 1036 fi 1037 # If the component doesn't run in a subshell, changing error_test_i to an 1038 # invalid integer will cause an error in the loop that runs this function. 1039 error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell 1040 # Expected error: 'grep non_existent /dev/null -> 1' 1041 grep non_existent /dev/null 1042 # Expected error: '! grep -q . tests/scripts/all.sh -> 1' 1043 not grep -q . "$0" 1044 # Expected error: 'make unknown_target -> 2' 1045 make unknown_target 1046 false "this should not be executed" 1047} 1048 1049# Run one component and clean up afterwards. 1050run_component () { 1051 current_component="$1" 1052 export MBEDTLS_TEST_CONFIGURATION="$current_component" 1053 1054 # Unconditionally create a seedfile that's sufficiently long. 1055 # Do this before each component, because a previous component may 1056 # have messed it up or shortened it. 1057 local dd_cmd 1058 dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1) 1059 case $OSTYPE in 1060 linux*|freebsd*|openbsd*) dd_cmd+=(status=none) 1061 esac 1062 "${dd_cmd[@]}" 1063 1064 # Run the component in a subshell, with error trapping and output 1065 # redirection set up based on the relevant options. 1066 if [ $KEEP_GOING -eq 1 ]; then 1067 # We want to keep running if the subshell fails, so 'set -e' must 1068 # be off when the subshell runs. 1069 set +e 1070 fi 1071 ( 1072 if [ $QUIET -eq 1 ]; then 1073 # msg() will be silenced, so just print the component name here. 1074 echo "${current_component#component_}" 1075 exec >/dev/null 1076 fi 1077 if [ $KEEP_GOING -eq 1 ]; then 1078 # Keep "set -e" off, and run an ERR trap instead to record failures. 1079 set -E 1080 trap err_trap ERR 1081 fi 1082 # The next line is what runs the component 1083 "$@" 1084 if [ $KEEP_GOING -eq 1 ]; then 1085 trap - ERR 1086 exit $last_failure_status 1087 fi 1088 ) 1089 component_status=$? 1090 if [ $KEEP_GOING -eq 1 ]; then 1091 set -e 1092 if [ $component_status -ne 0 ]; then 1093 failure_count=$((failure_count + 1)) 1094 fi 1095 fi 1096 1097 # Restore the build tree to a clean state. 1098 cleanup 1099 unset current_component 1100} 1101 1102# Preliminary setup 1103pre_check_environment 1104pre_initialize_variables 1105pre_parse_command_line "$@" 1106 1107setup_quiet_wrappers 1108pre_check_git 1109pre_restore_files 1110pre_back_up 1111 1112build_status=0 1113if [ $KEEP_GOING -eq 1 ]; then 1114 pre_setup_keep_going 1115fi 1116pre_prepare_outcome_file 1117pre_print_configuration 1118pre_check_tools 1119cleanup 1120if in_mbedtls_repo; then 1121 pre_generate_files 1122fi 1123 1124# Run the requested tests. 1125for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do 1126 run_component pseudo_component_error_test 1127done 1128unset error_test_i 1129for component in $RUN_COMPONENTS; do 1130 run_component "component_$component" 1131done 1132 1133# We're done. 1134post_report 1135
