1#!/bin/sh 2 3# basic-build-tests.sh 4# 5# This file is part of mbed TLS (https://tls.mbed.org) 6# 7# Copyright (c) 2016, ARM Limited, All Rights Reserved 8# 9# Purpose 10# 11# Executes the basic test suites, captures the results, and generates a simple 12# test report and code coverage report. 13# 14# The tests include: 15# * Unit tests - executed using tests/scripts/run-test-suite.pl 16# * Self-tests - executed using the test suites above 17# * System tests - executed using tests/ssl-opt.sh 18# * Interoperability tests - executed using tests/compat.sh 19# 20# The tests focus on functionality and do not consider performance. 21# 22# Note the tests self-adapt due to configurations in include/mbedtls/config.h 23# which can lead to some tests being skipped, and can cause the number of 24# available tests to fluctuate. 25# 26# This script has been written to be generic and should work on any shell. 27# 28# Usage: basic-build-tests.sh 29# 30 31# Abort on errors (and uninitiliased variables) 32set -eu 33 34if [ -d library -a -d include -a -d tests ]; then :; else 35 echo "Must be run from mbed TLS root" >&2 36 exit 1 37fi 38 39: ${OPENSSL:="openssl"} 40: ${OPENSSL_LEGACY:="$OPENSSL"} 41: ${GNUTLS_CLI:="gnutls-cli"} 42: ${GNUTLS_SERV:="gnutls-serv"} 43: ${GNUTLS_LEGACY_CLI:="$GNUTLS_CLI"} 44: ${GNUTLS_LEGACY_SERV:="$GNUTLS_SERV"} 45 46# To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh 47# we just export the variables they require 48export OPENSSL_CMD="$OPENSSL" 49export GNUTLS_CLI="$GNUTLS_CLI" 50export GNUTLS_SERV="$GNUTLS_SERV" 51 52CONFIG_H='include/mbedtls/config.h' 53CONFIG_BAK="$CONFIG_H.bak" 54 55# Step 0 - print build environment info 56OPENSSL="$OPENSSL" \ 57 OPENSSL_LEGACY="$OPENSSL_LEGACY" \ 58 GNUTLS_CLI="$GNUTLS_CLI" \ 59 GNUTLS_SERV="$GNUTLS_SERV" \ 60 GNUTLS_LEGACY_CLI="$GNUTLS_LEGACY_CLI" \ 61 GNUTLS_LEGACY_SERV="$GNUTLS_LEGACY_SERV" \ 62 scripts/output_env.sh 63echo 64 65# Step 1 - Make and instrumented build for code coverage 66export CFLAGS=' --coverage -g3 -O0 ' 67make clean 68cp "$CONFIG_H" "$CONFIG_BAK" 69scripts/config.pl full 70scripts/config.pl unset MBEDTLS_MEMORY_BACKTRACE 71make -j 72 73 74# Step 2 - Execute the tests 75TEST_OUTPUT=out_${PPID} 76cd tests 77 78# Step 2a - Unit Tests 79perl scripts/run-test-suites.pl -v |tee unit-test-$TEST_OUTPUT 80echo 81 82# Step 2b - System Tests 83sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT 84echo 85 86# Step 2c - Compatibility tests 87sh compat.sh -m 'tls1 tls1_1 tls1_2 dtls1 dtls1_2' | \ 88 tee compat-test-$TEST_OUTPUT 89OPENSSL_CMD="$OPENSSL_LEGACY" \ 90 sh compat.sh -m 'ssl3' |tee -a compat-test-$TEST_OUTPUT 91OPENSSL_CMD="$OPENSSL_LEGACY" \ 92 GNUTLS_CLI="$GNUTLS_LEGACY_CLI" \ 93 GNUTLS_SERV="$GNUTLS_LEGACY_SERV" \ 94 sh compat.sh -e '3DES\|DES-CBC3' -f 'NULL\|DES\|RC4\|ARCFOUR' | \ 95 tee -a compat-test-$TEST_OUTPUT 96echo 97 98# Step 3 - Process the coverage report 99cd .. 100make lcov |tee tests/cov-$TEST_OUTPUT 101 102 103# Step 4 - Summarise the test report 104echo 105echo "=========================================================================" 106echo "Test Report Summary" 107echo 108 109cd tests 110 111# Step 4a - Unit tests 112echo "Unit tests - tests/scripts/run-test-suites.pl" 113 114PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') 115SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') 116TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') 117FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') 118 119echo "No test suites : $TOTAL_SUITES" 120echo "Passed : $PASSED_TESTS" 121echo "Failed : $FAILED_TESTS" 122echo "Skipped : $SKIPPED_TESTS" 123echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" 124echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" 125echo 126 127TOTAL_PASS=$PASSED_TESTS 128TOTAL_FAIL=$FAILED_TESTS 129TOTAL_SKIP=$SKIPPED_TESTS 130TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) 131TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) 132 133# Step 4b - TLS Options tests 134echo "TLS Options tests - tests/ssl-opt.sh" 135 136PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') 137SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') 138TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') 139FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) 140 141echo "Passed : $PASSED_TESTS" 142echo "Failed : $FAILED_TESTS" 143echo "Skipped : $SKIPPED_TESTS" 144echo "Total exec'd tests : $TOTAL_TESTS" 145echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" 146echo 147 148TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 149TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 150TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 151TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) 152TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) 153 154 155# Step 4c - System Compatibility tests 156echo "System/Compatibility tests - tests/compat.sh" 157 158PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 159SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 160EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 161FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) 162 163echo "Passed : $PASSED_TESTS" 164echo "Failed : $FAILED_TESTS" 165echo "Skipped : $SKIPPED_TESTS" 166echo "Total exec'd tests : $EXED_TESTS" 167echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" 168echo 169 170TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 171TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 172TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 173TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) 174TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) 175 176 177# Step 4d - Grand totals 178echo "-------------------------------------------------------------------------" 179echo "Total tests" 180 181echo "Total Passed : $TOTAL_PASS" 182echo "Total Failed : $TOTAL_FAIL" 183echo "Total Skipped : $TOTAL_SKIP" 184echo "Total exec'd tests : $TOTAL_EXED" 185echo "Total avail tests : $TOTAL_AVAIL" 186echo 187 188 189# Step 4e - Coverage 190echo "Coverage" 191 192LINES_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* lines)/\1/p') 193LINES_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ lines......: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) lines)/\1/p') 194FUNCS_TESTED=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% (\([0-9]*\) of [0-9]* functions)$/\1/p') 195FUNCS_TOTAL=$(tail -n3 cov-$TEST_OUTPUT|sed -n -e 's/ functions..: [0-9]*.[0-9]% ([0-9]* of \([0-9]*\) functions)$/\1/p') 196 197LINES_PERCENT=$((1000*$LINES_TESTED/$LINES_TOTAL)) 198LINES_PERCENT="$(($LINES_PERCENT/10)).$(($LINES_PERCENT-($LINES_PERCENT/10)*10))" 199 200FUNCS_PERCENT=$((1000*$FUNCS_TESTED/$FUNCS_TOTAL)) 201FUNCS_PERCENT="$(($FUNCS_PERCENT/10)).$(($FUNCS_PERCENT-($FUNCS_PERCENT/10)*10))" 202 203echo "Lines Tested : $LINES_TESTED of $LINES_TOTAL $LINES_PERCENT%" 204echo "Functions Tested : $FUNCS_TESTED of $FUNCS_TOTAL $FUNCS_PERCENT%" 205echo 206 207 208rm unit-test-$TEST_OUTPUT 209rm sys-test-$TEST_OUTPUT 210rm compat-test-$TEST_OUTPUT 211rm cov-$TEST_OUTPUT 212 213cd .. 214 215make clean 216 217if [ -f "$CONFIG_BAK" ]; then 218 mv "$CONFIG_BAK" "$CONFIG_H" 219fi 220
