1#!/bin/sh 2 3# basic-build-test.sh 4# 5# Copyright The Mbed TLS Contributors 6# SPDX-License-Identifier: Apache-2.0 7# 8# Licensed under the Apache License, Version 2.0 (the "License"); you may 9# not use this file except in compliance with the License. 10# You may obtain a copy of the License at 11# 12# http://www.apache.org/licenses/LICENSE-2.0 13# 14# Unless required by applicable law or agreed to in writing, software 15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17# See the License for the specific language governing permissions and 18# limitations under the License. 19# 20# Purpose 21# 22# Executes the basic test suites, captures the results, and generates a simple 23# test report and code coverage report. 24# 25# The tests include: 26# * Unit tests - executed using tests/scripts/run-test-suite.pl 27# * Self-tests - executed using the test suites above 28# * System tests - executed using tests/ssl-opt.sh 29# * Interoperability tests - executed using tests/compat.sh 30# 31# The tests focus on functionality and do not consider performance. 32# 33# Note the tests self-adapt due to configurations in include/mbedtls/mbedtls_config.h 34# which can lead to some tests being skipped, and can cause the number of 35# available tests to fluctuate. 36# 37# This script has been written to be generic and should work on any shell. 38# 39# Usage: basic-build-test.sh 40# 41 42# Abort on errors (and uninitiliased variables) 43set -eu 44 45if [ -d library -a -d include -a -d tests ]; then :; else 46 echo "Must be run from mbed TLS root" >&2 47 exit 1 48fi 49 50: ${OPENSSL:="openssl"} 51: ${OPENSSL_LEGACY:="$OPENSSL"} 52: ${GNUTLS_CLI:="gnutls-cli"} 53: ${GNUTLS_SERV:="gnutls-serv"} 54: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} 55: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} 56 57# Used to make ssl-opt.sh deterministic. 58# 59# See also RELEASE_SEED in all.sh. Debugging is easier if both values are kept 60# in sync. If you change the value here because it breaks some tests, you'll 61# definitely want to change it in all.sh as well. 62: ${SEED:=1} 63export SEED 64 65# if MAKEFLAGS is not set add the -j option to speed up invocations of make 66if [ -z "${MAKEFLAGS+set}" ]; then 67 export MAKEFLAGS="-j" 68fi 69 70# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh 71# we just export the variables they require 72export OPENSSL="$OPENSSL" 73export GNUTLS_CLI="$GNUTLS_CLI" 74export GNUTLS_SERV="$GNUTLS_SERV" 75 76CONFIG_H='include/mbedtls/mbedtls_config.h' 77CONFIG_BAK="$CONFIG_H.bak" 78 79# Step 0 - print build environment info 80OPENSSL="$OPENSSL" \ 81 OPENSSL_LEGACY="$OPENSSL_LEGACY" \ 82 GNUTLS_CLI="$GNUTLS_CLI" \ 83 GNUTLS_SERV="$GNUTLS_SERV" \ 84 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ 85 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \ 86 scripts/output_env.sh 87echo 88 89# Step 1 - Make and instrumented build for code coverage 90export CFLAGS=' --coverage -g3 -O0 ' 91export LDFLAGS=' --coverage' 92make clean 93cp "$CONFIG_H" "$CONFIG_BAK" 94scripts/config.py full 95make 96 97 98# Step 2 - Execute the tests 99TEST_OUTPUT=out_${PPID} 100cd tests 101if [ ! -f "seedfile" ]; then 102 dd if=/dev/urandom of="seedfile" bs=64 count=1 103fi 104echo 105 106# Step 2a - Unit Tests (keep going even if some tests fail) 107echo '################ Unit tests ################' 108perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT 109echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^' 110echo 111 112# Step 2b - System Tests (keep going even if some tests fail) 113echo 114echo '################ ssl-opt.sh ################' 115echo "ssl-opt.sh will use SEED=$SEED for udp_proxy" 116sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT 117echo '^^^^^^^^^^^^^^^^ ssl-opt.sh ^^^^^^^^^^^^^^^^' 118echo 119 120# Step 2c - Compatibility tests (keep going even if some tests fail) 121echo '################ compat.sh ################' 122{ 123 echo '#### compat.sh: Default versions' 124 sh compat.sh 125 echo 126 127 echo '#### compat.sh: legacy (null)' 128 OPENSSL="$OPENSSL_LEGACY" \ 129 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ 130 sh compat.sh -e '^$' -f 'NULL' 131 echo 132 133 echo '#### compat.sh: next (ARIA, ChaCha)' 134 OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' 135 echo 136} | tee compat-test-$TEST_OUTPUT 137echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^' 138echo 139 140# Step 3 - Process the coverage report 141cd .. 142{ 143 make lcov 144 echo SUCCESS 145} | tee tests/cov-$TEST_OUTPUT 146 147if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then 148 echo >&2 "Fatal: 'make lcov' failed" 149 exit 2 150fi 151 152 153# Step 4 - Summarise the test report 154echo 155echo "=========================================================================" 156echo "Test Report Summary" 157echo 158 159# A failure of the left-hand side of a pipe is ignored (this is a limitation 160# of sh). We'll use the presence of this file as a marker that the generation 161# of the report succeeded. 162rm -f "tests/basic-build-test-$$.ok" 163 164{ 165 166 cd tests 167 168 # Step 4a - Unit tests 169 echo "Unit tests - tests/scripts/run-test-suites.pl" 170 171 PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') 172 SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') 173 TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') 174 FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') 175 176 echo "No test suites : $TOTAL_SUITES" 177 echo "Passed : $PASSED_TESTS" 178 echo "Failed : $FAILED_TESTS" 179 echo "Skipped : $SKIPPED_TESTS" 180 echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" 181 echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" 182 echo 183 184 TOTAL_PASS=$PASSED_TESTS 185 TOTAL_FAIL=$FAILED_TESTS 186 TOTAL_SKIP=$SKIPPED_TESTS 187 TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) 188 TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) 189 190 # Step 4b - TLS Options tests 191 echo "TLS Options tests - tests/ssl-opt.sh" 192 193 PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') 194 SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') 195 TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') 196 FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) 197 198 echo "Passed : $PASSED_TESTS" 199 echo "Failed : $FAILED_TESTS" 200 echo "Skipped : $SKIPPED_TESTS" 201 echo "Total exec'd tests : $TOTAL_TESTS" 202 echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" 203 echo 204 205 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 206 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 207 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 208 TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) 209 TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) 210 211 212 # Step 4c - System Compatibility tests 213 echo "System/Compatibility tests - tests/compat.sh" 214 215 PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 216 SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 217 EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 218 FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) 219 220 echo "Passed : $PASSED_TESTS" 221 echo "Failed : $FAILED_TESTS" 222 echo "Skipped : $SKIPPED_TESTS" 223 echo "Total exec'd tests : $EXED_TESTS" 224 echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" 225 echo 226 227 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 228 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 229 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 230 TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) 231 TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) 232 233 234 # Step 4d - Grand totals 235 echo "-------------------------------------------------------------------------" 236 echo "Total tests" 237 238 echo "Total Passed : $TOTAL_PASS" 239 echo "Total Failed : $TOTAL_FAIL" 240 echo "Total Skipped : $TOTAL_SKIP" 241 echo "Total exec'd tests : $TOTAL_EXED" 242 echo "Total avail tests : $TOTAL_AVAIL" 243 echo 244 245 246 # Step 4e - Coverage report 247 echo "Coverage statistics:" 248 sed -n '1,/^Overall coverage/d; /%/p' cov-$TEST_OUTPUT 249 echo 250 251 rm unit-test-$TEST_OUTPUT 252 rm sys-test-$TEST_OUTPUT 253 rm compat-test-$TEST_OUTPUT 254 rm cov-$TEST_OUTPUT 255 256 # Mark the report generation as having succeeded. This must be the 257 # last thing in the report generation. 258 touch "basic-build-test-$$.ok" 259} | tee coverage-summary.txt 260 261make clean 262 263if [ -f "$CONFIG_BAK" ]; then 264 mv "$CONFIG_BAK" "$CONFIG_H" 265fi 266 267# The file must exist, otherwise it means something went wrong while generating 268# the coverage report. If something did go wrong, rm will complain so this 269# script will exit with a failure status. 270rm "tests/basic-build-test-$$.ok" 271
