parallel-tests: Ensure backward-compatible semantics. For each test in Automake's test suite that uses TESTS, generate an identical one that uses the `parallel-tests' option, for coverage of backward-compatible functionality. * tests/gen-parallel-tests: New file, generates distributed Makefile.am snippet tests/parallel-tests.am to list all tests that use the TESTS interface but not yet the `parallel-tests' option, with names mangled to use suffix `-p.test', in ... (parallel_tests): ... this new make macro. * tests/Makefile.am ($(srcdir)/parallel-tests.am) ($(parallel_tests), defs-p): New rules. (TESTS): Add $(parallel_tests). (check_SCRIPTS): Add defs-p, $(parallel_tests). (check-clean-local): Remove `defs-p'. (EXTRA_DIST): Distribute gen-parallel-tests. * bootstrap: Generate parallel-tests.am. * tests/check8.test: Check for circular dependencies in rules. * tests/check11.test: New test, check that SKIPs are not counted as passed tests. * tests/defs.in: Unset DISABLE_HARD_ERRORS, LAZY_TEST_SUITE, VERBOSE, so the tests are not influenced by the way our test suite is invoked.
diff --git a/bootstrap b/bootstrap index aa50f5d..1abde1d 100755 --- a/bootstrap +++ b/bootstrap @@ -122,6 +122,11 @@ dosubst m4/amversion.in m4/amversion.m4 # Create temporary replacement for automake dosubst automake.in automake.tmp +# Create tests/parallel-tests.am. +cd tests +$BOOTSTRAP_SHELL ./gen-parallel-tests > parallel-tests.am +cd .. + # Run the autotools. $PERL ./aclocal.tmp -I m4 autoconf diff --git a/tests/.gitignore b/tests/.gitignore index a6827fd..cfd4565 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1,6 +1,9 @@ aclocal-* automake-* defs +defs-p +parallel-tests.am *.dir *.log *.log-t +*-p.test diff --git a/tests/Makefile.am b/tests/Makefile.am index 9cc6dbe..b56e7e0 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -7,6 +7,19 @@ auxdir2.test \ cond17.test \ txinfo5.test +include $(srcdir)/parallel-tests.am + +$(srcdir)/parallel-tests.am: gen-parallel-tests Makefile.am + (cd $(srcdir) && $(SHELL) ./gen-parallel-tests) >$@ + +$(parallel_tests): $(parallel_tests:-p.test=.test) Makefile.am + input=`echo $@ | sed 's,.*/,,; s,-p.test$$,.test,'`; \ + sed 's,^\. \./defs,. ./defs-p,' < $(srcdir)/$$input > $@ + chmod a+rx $@ + +defs-p: defs Makefile.am + sed 's,^AM_INIT_AUTOMAKE$$,&([parallel-tests]),' < defs >$@ + TESTS = \ aclibobj.test \ aclocal.test \ @@ -98,6 +111,7 @@ check7.test \ check8.test \ check9.test \ check10.test \ +check11.test \ checkall.test \ clean.test \ clean2.test \ @@ -664,15 +678,16 @@ yacc6.test \ yacc7.test \ yacc8.test \ yaccpp.test \ -yaccvpath.test +yaccvpath.test \ +$(parallel_tests) -EXTRA_DIST = ChangeLog-old $(TESTS) +EXTRA_DIST = ChangeLog-old gen-parallel-tests $(TESTS) # Each test case depends on defs, aclocal, and automake. -check_SCRIPTS = defs aclocal-$(APIVERSION) automake-$(APIVERSION) +check_SCRIPTS = defs defs-p aclocal-$(APIVERSION) automake-$(APIVERSION) $(parallel_tests) clean-local: check-clean-local check-clean-local: -chmod -R u+rwx *.dir - -rm -rf *.dir + -rm -rf defs-p *.dir diff --git a/tests/check11.test b/tests/check11.test new file mode 100755 index 0000000..d4f6edc --- /dev/null +++ b/tests/check11.test @@ -0,0 +1,53 @@ +#! /bin/sh +# Copyright (C) 2008 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Check skip summary. + +. ./defs || Exit 1 + +set -e + +cat >> configure.in << 'END' +AC_OUTPUT +END + +cat > Makefile.am << 'END' +TESTS = skip skip2 +END + +cat >>skip <<'END' +#! /bin/sh +exit 77 +END +chmod a+x skip +cp skip skip2 + +$ACLOCAL +$AUTOCONF +$AUTOMAKE -a + +unset TESTS || : + +./configure +env TESTS=skip $MAKE -e check >stdout +cat stdout +grep '1.*passed' stdout && Exit 1 + +env TESTS="skip skip2" $MAKE -e check >stdout +cat stdout +grep '2.*passed' stdout && Exit 1 + +: diff --git a/tests/check8.test b/tests/check8.test index 242be56..3e38ccf 100755 --- a/tests/check8.test +++ b/tests/check8.test @@ -67,14 +67,20 @@ $AUTOMAKE -a unset TESTS || : ./configure -AM_COLOR_TESTS=always $MAKE -e check >stdout && { cat stdout; Exit 1; } +AM_COLOR_TESTS=always $MAKE -e check >stdout 2>stderr && + { cat stdout; cat stderr >&2; Exit 1; } cat stdout +cat stderr >&2 grep 'XPASS.* foo$' stdout grep '^[^X]*PASS.* sub/foo$' stdout grep '^[^X]*PASS.* bar$' stdout grep '^[^X]*PASS.* sub/bar$' stdout grep '^[^X]*FAIL.* baz$' stdout grep 'XFAIL.* sub/baz$' stdout +# parallel-tests should not add circular dependencies. +# Look for known warnings from a couple of `make' implementations. +grep -i 'circular.*dependency' stderr && Exit 1 +grep -i 'graph cycles' stderr && Exit 1 $MAKE distclean diff --git a/tests/defs.in b/tests/defs.in index 063ce58..1e91183 100644 --- a/tests/defs.in +++ b/tests/defs.in @@ -290,6 +290,13 @@ unset MFLAGS unset MAKEFLAGS unset MAKELEVEL unset DESTDIR +# Also unset variables that control our test driver. While not +# conceptually independent, they cause some changed semantics we +# need to control (and test for) in some of the tests to ensure +# backward-compatible behavior. +unset DISABLE_HARD_ERRORS +unset LAZY_TEST_SUITE +unset VERBOSE echo "=== Running test $0" diff --git a/tests/gen-parallel-tests b/tests/gen-parallel-tests new file mode 100755 index 0000000..4a71a73 --- /dev/null +++ b/tests/gen-parallel-tests @@ -0,0 +1,26 @@ +#! /bin/sh +# Generate parallel-tests.am. +# +# For each test in the TESTS list in this Makefile.am file, that itself +# tests features of the TESTS automake interface, generate a sibling +# test that does likewise, but with the option `parallel-tests' enabled. + +set -e + +tests=`sed -n '/^TESTS =/,/^$/s/\(.*\.test\).*/\1/p' Makefile.am` +{ + grep -l '^TESTS ' $tests + grep -l ' TESTS ' $tests +} | +grep -v '.-p\.test' | +LC_ALL=C sort -u | +while read tst; do + if grep '^[^#]*parallel-tests' $tst >/dev/null \ + || grep '^\. \./defs-p' $tst >/dev/null + then :; else echo $tst; fi; +done | +{ + echo "## Generated by gen-parallel-tests. Edit Makefile.am instead of this." + echo "parallel_tests = \\" + sed 's,\.test$,-p.test,; $!s,$, \\,' +}