Hello,
When working on SourceLayout I got a few false errors from
build-test.sh when it could not find some string in the make output. The
attached patch fixes that by relying on make exit code rather than some
magic string some make targets may print in the middle of the make
process. The build is still considered failed if error strings are found
in the log.
I was also annoyed on how the test would quit on the first error,
wasting a lot of test time if the first error gets unnoticed. The
attached patch adds a --keep-going option, similar to "make -k". The
option is off by default so I am not changing the default behavior here.
Finally, I made test spec names processing a little bit more flexible.
It is now possible to explicitly specify multiple tests, similar to
"make all check distcheck".
More detailed comments are below. Please review.
Thank you,
Alex.
bb:approve
------- change log -----------
Support multiple test spec arguments. If at least one test fails, the script
exits with a non-zero code (but possibly not immediately, see --keep-going).
Each test spec is a test config file name or a well-known config name
(no path
or extension!). If no specs are given, all known test specs are used (as
before). The same happens if the only test spec given is 'all'. The
following are now equivalent:
./test-builds.sh
./test-builds.sh all
./test-builds.sh btlayer-00-default btlayer-01-minimal
btlayer-02-maximus
./test-builds.sh test-suite/buildtests/layer-*
You can mix file names and spec names, but not the 'all' macro: There is
currently no support for using 'all' together with other test cases.
Tolerate individual test errors if --keep-going is specified. This helps
when
one wants to find more errors than just the first one, especially when tests
are long and are running without a human watching.
When detecting test failures, rely on test-suite/buildtest.sh exit
status code
rather than on the presence of error-like strings in the log file.
Added and polished comments.
-----------------------------------
# Bazaar merge directive format 2 (Bazaar 0.90)
# revision_id: [email protected]\
# 6lkaoiv1jznsmql6
# target_branch: http://www.squid-cache.org/bzr/squid3/trunk
# testament_sha1: 66c28c9b4c5a70895ab7db67ca0763657272c2b7
# timestamp: 2009-03-09 10:38:20 -0600
# base_revision_id: [email protected]\
# 4hkp9exkjcdwqvfi
#
# Begin patch
=== modified file 'test-builds.sh'
--- test-builds.sh 2009-03-08 10:57:37 +0000
+++ test-builds.sh 2009-03-09 16:20:57 +0000
@@ -1,11 +1,14 @@
#!/bin/sh
#
-# Run specific build tests for a given OS environment.
+# Run all or some build tests for a given OS environment.
#
top=`dirname $0`
+globalResult=0
+
cleanup="no"
verbose="no"
+keepGoing="no"
while [ $# -ge 1 ]; do
case "$1" in
--cleanup)
@@ -16,15 +19,16 @@
verbose="yes"
shift
;;
+ --keep-going)
+ keepGoing="yes"
+ shift
+ ;;
*)
break
;;
esac
done
-# Things to catch
-errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:"
-
logtee() {
if [ $verbose = yes ]; then
tee $1
@@ -41,52 +45,93 @@
echo "TESTING: ${layer}"
rm -f -r ${btlayer} && mkdir ${btlayer}
{
+ result=255
cd ${btlayer}
if test -e $top/test-suite/buildtest.sh ; then
$top/test-suite/buildtest.sh $opts
+ result=$?
elif test -e ../$top/test-suite/buildtest.sh ; then
../$top/test-suite/buildtest.sh ../$opts
+ result=$?
+ else
+ echo "Error: cannot find $top/test-suite/buildtest.sh script"
+ result=1
fi
+ # log the result for the outer script to notice
+ echo "buildtest.sh result is $result";
} 2>&1 | logtee $log
+
+ result=1 # failure by default
+ if grep -q '^buildtest.sh result is 0$' $log; then
+ result=0
+ fi
+
grep -E "BUILD" ${log}
- grep -E "${errors}" $log && exit 1
+
+ # logged strings to treat as errors
+ errors="^ERROR|\ error:|\ Error\ |No\ such|assertion\ failed|FAIL:"
+ if grep -E "${errors}" $log; then
+ # Possible errors detected.
+ # Let's be conservative and assume those were real errors.
+ if test $result -eq 0; then
+ echo "Internal error: failed test with a successful result code"
+ result=1
+ # else we already know that there was an error
+ fi
+ fi
+
if test "${cleanup}" = "yes" ; then
echo "REMOVE DATA: ${btlayer}"
rm -f -r ${btlayer}
fi
- result=`tail -2 $log | head -1`
- if test "${result}" = "Build Successful." ; then
- echo "${result}"
+
+ if test $result -eq 0; then
+ # successful execution
+ if test "$verbose" = yes; then
+ echo 'Build OK.'
+ fi
else
- echo "Build Failed:"
+ echo "Build Failed ($result):"
tail -5 $log
- exit 1
+ globalResult=1
fi
+
if test "${cleanup}" = "yes" ; then
echo "REMOVE LOG: ${log}"
rm -f -r $log
fi
}
-# Run a single test build by name or opts file
-if [ -e "$1" ]; then
-
- buildtest $1
- exit 0
-fi
-tmp=`basename "${1}" .opts`
-if test -e $top/test-suite/buildtests/${tmp}.opts ; then
- buildtest $top/test-suite/buildtests/${tmp}.opts
- exit 0
-fi
-
-#
-# Run specific tests for each combination of configure-time
-# Options.
-#
-# These layers are constructed from detailed knowledge of
-# component dependencies.
-#
-for f in `ls -1 $top/test-suite/buildtests/layer*.opts` ; do
- buildtest $f
+# decide what tests to run, $* contains test spec names or filenames
+# use all knows specs if $* is empty
+if test -n "$*" -a "$*" != all; then
+ tests="$*"
+else
+ tests=`ls -1 $top/test-suite/buildtests/layer*.opts`
+fi
+
+for t in $tests; do
+ if test -e "$t"; then
+ # A configuration file
+ cfg="$t"
+ elif test -e "$top/test-suite/buildtests/${t}.opts"; then
+ # A well-known configuration name
+ cfg="$top/test-suite/buildtests/${t}.opts"
+ else
+ echo "Error: Unknown test specs '$t'"
+ cfg=''
+ globalResult=1
+ fi
+
+ # run the test, if any
+ if test -n "$cfg"; then
+ buildtest $cfg
+ fi
+
+ # quit on errors unless we should $keepGoing
+ if test $globalResult -ne 0 -a $keepGoing != yes; then
+ exit $globalResult
+ fi
done
+
+exit $globalResult
# Begin bundle
IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWXjpNacAA49fgEQwfff///Ot
3qX////+YAk9X27tgtHHKt0dcddBo7btqJbZRThKIQmmIZGhpkJtTIT9RNpPTUHqeoPUaaeo0DQB
ojQJhNAmptTUwQA9RoAaGjQaAaaAONGTIwjEAwmgwCaDQMmTRkyGEBhKZIIRpNI9U9qeomymj1P1
TxR6GmmhNMgNAeoDQONGTIwjEAwmgwCaDQMmTRkyGEBhIiIATQmTU9NEyGUn6UYYU00NA0AAHlKA
d0REFYgKogQkj2ba4/iK72Km19qwujlnueFpjYZ2OtdxS2ppXnGw3FSlvMaXaJYnYSNbjmMb9Lch
O7FeEvd+fbkLZf8s9llOOr+3t8BgZaPg4ooKLpNbpYVEG9iUMNHp+GGNOrhlgySjhUpNnCuvKs7e
ONe0SVGCIkYGYURFVCuXF3QEJf2UIXOji8HGGLlcZI34fy6PRS1HBbjmadE9JkS1hn2YZGZETaSH
EANnU2eILtZSMlaOJqphUsmuQdPMitqFVGKurHHM9agrg5ywhlqOOHmGsmZ41vHUq+WfUY2srpi9
LH87Ljc6uazNE151t3TaifTRjuJVlFH6Tf4Mu7BnGi+f9Mvu0bGZtZetWGeEHzOGrQ917NUqLHS3
Hbo7AG9DGBwGtTIGlpw3LUlkGuvB7WyYp7HxVEWpab7mhS4KCxbTUvB9HJU73qeLaGGwYVjXrXAC
yKnGo2I+mTnyABpN9x1pOomxSGulQ2OaBs0VG5WsEGysFizGNysGxMQNL9Fhd52OpsGVm8GwpDdt
YttB/lsshfns01WNssV7BMiqqoqF63Ox+zlfgBQVUDX5QSldkKbUPiiw2iD1s78jEj2H+T01+roN
dcpcDVQlwrvyPe2FDTymN99mpXosSsRcp/L6BPShoEfSJBm0yOBwh6EmTCScnngS6toTB2X597KH
2Dalf78YTYLVAh02FkaPAol4uiMOCEsLqwoCms6PVRGCFil9Vdvd0BMj0iLaCYrzxLiAvKqREigw
LMD8nfJObTTCrAjOM3XPCaYx+Fhpgu2Sv2kHQ9m7dfwuP+mXwHU2VtptFIddi4WjuPvFYFrModre
/gCvCjIBicIiMBhKozxC8Yi1wFx1pR4Gq5al5hoIKgH5RnhyDcaN4QzMHIxawkqRsMkBmNYuLsCq
XVxD1D4ZSecFHYK1qpQZUdXpwiiksUVKFtCRSchgxlXJrixLQGLwVs4AoHMGSYBxGIige2QlUVNW
KjqIVWqKS4xlQZFt9oK5iwjNwOB7DnU7lXqzpz8b0BNImBLw5SVB6HbTlgVmVrGtqbe7V7UVROCm
zvLHE49qwNVdhS3h50SpdxoC2rrOiZszBTPWcxByXjvmgWBVReDdk/O94q8Q/5cQb2Dvt4NfX8KL
JmEaPkmRoRqpaMPRlkmcl9Bce7JUN7cZcEfDqOsUj82s3xY6t/wEJjWSk4aHkIYJ7XylE07pwoRW
LASKiW24dqx3UAT4K5gTda98yyIHzOVIQ4F4kQmaohYdOsVFfXIbX6kN241FBo0Qm0eOYRS/RNkW
a28GlCY2dj3WxB/NAyJIOxBjK9RU+Sp30+48D3n9dTfbduzFg1JV/q0TEMDyVKiYjWDuwggcDGmQ
ybqL4kqJCvM7AtPj0LH0M3Ujrzgn2GXEpmAYYUV12vqdHb5Z5kO19N9++vi+TQ1k6bnKoLOaolJG
HNovS1qLQZic2wE9p4rrK/TgWfiN1/oKUDTIm0RHJHganf3n6260ZCQ9gsxZLOjHZNFwqyHIgqLs
yI3iQ4xpfoLsV8PBZgYiLEjtbb3btvXd1JFb7VJCL2WpdxNuC+CN/SmONm16VsvEa/PUsY9VmPWZ
PQTdpYcTJCD/kwC+4tGwG0xn7PDqO6gGGJy9ER61ipy2L9HlBHSEHQRYiReR4lTgjmGj0juVeoMP
3NWNExDZi1Xme/K1FNYhw5a9pJJI/ZqLlUwX0rPU3qhgf456GvehQlHEgQgZflfIohI2VrMBVe9u
LKt5++GKlHbPADUQOmVPTabK19xPu+YXduCuMl3HFtnq2RTYX+d4W5AiEuibWmRf9TVTuCM+06uu
Lr4M2cD15/D145YZMYNg7lx2VFk91ANpubFtCNmoqY+CTvXLeBHE5JwEYPCZ9CVk9MdQTJFAUsLq
IYUjpTAyE3BT9yEOSPQcUW4YIPIo6UNaEF92fZz66V6iElQc4x0Fgchmttf85yFyF7WIsdDbrZrZ
GYjG1/EVZhphDHSCGkAQipjAMutEWNl5UkFv2iWsEjVlFkGJI9NOUnGRsakc1dat0cBrwglRzjtd
jY0X2hRhINpHxpG35Y2Du9uJ+l2sPMWWF8aSEEPqYUycQ4T9fT9G8ju8ZqSEvnn38hExPDIWf/jx
TMxe/z0wEdrNxixj0YEfU+NQznZI9gMN6eNbmQpFUPCQkDNUQt/jPy2K/0R87fz4CIs6JFhtaOxF
4UWyQs+j6PJTU5JGt4cqynz9SjDAWq8YRly5jI04zvi0GmWIQiF2sYJQbX3+AapaUJCeeO46ncg2
4QeInZ0lgR5o7Elt4K7VE5a89DwW1UInlXQNqS/v59hoQnvxP5DGwM0sWoTMVI1ECZYudrhaeowt
iDne9rXqam/uQZuoxwarZijzSRnVydEQg0ikVE0JFzWF41LWObbLZ5M3pFDrNvO+zHGuDStqkZqg
rRZNGELJmGCRMiT0gUtILGWBBAhrvFCEYTS0Rb5ardabBQSIVtCcpBpeDykrS17jRDpeXg1x4I0u
gtBDE5gI6qTaQI+0xsqFTEVbjrpNu67ZxYI5h2RAOQiZUhs9JJsx0UiRunZJShvJNdlpS6O9bLUZ
BuCbxDKe6hLG8uBJKTMZhUwmh9hq4cBL0+44EgByQtxZSZKEgDfzPwc+Z3FT/VU4wIfC/pcs0j4v
Ush6RwFSBEZsOK5b1M8wLDFIbBhxTmv3kGLSLdMsr51FKRDtR2mRc2I0a5hJM0yXLddGTHKf8Xck
U4UJB46TWnA=
