[PATCH v2] test:Improve test behaviors when --root is used
Change add_email_corpus, emacs_deliver_message and tests to use $TEST_DIRECTORY instead of '..'. This improves the behavior of the usage of --root=, as the assumption of what '..' means will usually be incorrect. Document -root option in README and update valgrind to work with -root. Signed-off-by: Mark Anderson --- > If you could follow up with an updated patch, (or an argument that the > original patch is correct), that would be great. Updated the patch with Austin's suggestion. I'm not personally ready to forbid the use of '..', although I certainly appreciate the motivation to keep our tests working in --root mode. test/README|9 + test/basic | 10 +- test/crypto|2 +- test/emacs |4 ++-- test/symbol-hiding |4 ++-- test/test-lib.sh | 18 +- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/test/README b/test/README index be75e0e..8fbf78d 100644 --- a/test/README +++ b/test/README @@ -41,6 +41,15 @@ The following command-line options are available when running tests: As the names depend on the tests' file names, it is safe to run the tests with this option in parallel. +--root=:: + This runs the testsuites specified under a seperate directory. + However, caution is advised, as not all tests are maintained + with this relocation in mind, so some tests may behave + differently. + + Pointing this argument at a tmpfs filesystem can improve the + speed of the test suite for some users. + When invoking the test suite via "make test" any of the above options can be specified as follows: diff --git a/test/basic b/test/basic index d6e8c10..33bf711 100755 --- a/test/basic +++ b/test/basic @@ -51,9 +51,9 @@ test_expect_code 2 'failure to clean up causes the test to fail' ' # Ensure that all tests are being run test_begin_subtest 'Ensure that all available tests will be run by notmuch-test' -eval $(sed -n -e '/^TESTS="$/,/^"$/p' notmuch-test ../notmuch-test) +eval $(sed -n -e '/^TESTS="$/,/^"$/p' notmuch-test $TEST_DIRECTORY/notmuch-test) tests_in_suite=$(for i in $TESTS; do echo $i; done | sort) -available=$(ls -1 ../ | \ +available=$(ls -1 $TEST_DIRECTORY/ | \ sed -r -e "/^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test)/d" \ -e "/^(README|test-lib.sh|test-lib.el|test-results|tmp.*|valgrind|corpus*)/d" \ -e "/^(emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|symbol-test.cc)/d" \ @@ -63,19 +63,19 @@ available=$(ls -1 ../ | \ | sort) test_expect_equal "$tests_in_suite" "$available" -EXPECTED=../test.expected-output +EXPECTED=$TEST_DIRECTORY/test.expected-output suppress_diff_date() { sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \ -e 's/\(.*\+\+\+ test-verbose\.4\.\output\).*/\1/' } test_begin_subtest "Ensure that test output is suppressed unless the test fails" -output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date) +output=$(cd $TEST_DIRECTORY; ./test-verbose 2>&1 | suppress_diff_date) expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date) test_expect_equal "$output" "$expected" test_begin_subtest "Ensure that -v does not suppress test output" -output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date) +output=$(cd $TEST_DIRECTORY; ./test-verbose -v 2>&1 | suppress_diff_date) expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date) # Do not include the results of test-verbose in totals rm $TEST_DIRECTORY/test-results/test-verbose-* diff --git a/test/crypto b/test/crypto index 01daffe..7eb3559 100755 --- a/test/crypto +++ b/test/crypto @@ -12,7 +12,7 @@ add_gnupg_home () local output [ -d ${GNUPGHOME} ] && return mkdir -m 0700 "$GNUPGHOME" -gpg --no-tty --import <../gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 +gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 test_debug "cat $GNUPGHOME/import.log" if (gpg --quick-random --version >/dev/null 2>&1) ; then echo quick-random >> "$GNUPGHOME"/gpg.conf diff --git a/test/emacs b/test/emacs index 6f82b08..f91078e 100755 --- a/test/emacs +++ b/test/emacs @@ -2,7 +2,7 @@ test_description="emacs interface" . test-lib.sh -EXPECTED=../emacs.expected-output +EXPECTED=$TEST_DIRECTORY/emacs.expected-output add_email_corpus @@ -81,7 +81,7 @@ mkdir -p mail/sent/cur mkdir -p mail/sent/new mkdir -p mail/sent/tmp -../smtp-dummy sent_message & +$TEST_DIRECTORY/smtp-dummy sent_message & smtp_dummy_pid=$! test_emacs "(setq message-send-mail-function 'message-smtpmail-send-it) (setq smtpmail-smtp-server \"localhost\") (setq smtpmail-smtp-service \"25025\") (notmuch-hello) (notmuch-mua-mail) (message-goto-to) (insert \"user at example.com\nDate: Fri, 29 Mar 1974 10:00:00 -\") (message-goto-subject) (insert \"Testing message sent via SMTP\") (message-goto-body) (insert \"This is a test that me
[PATCH v2] test:Improve test behaviors when --root is used
Change add_email_corpus, emacs_deliver_message and tests to use $TEST_DIRECTORY instead of '..'. This improves the behavior of the usage of --root=, as the assumption of what '..' means will usually be incorrect. Document -root option in README and update valgrind to work with -root. Signed-off-by: Mark Anderson --- > If you could follow up with an updated patch, (or an argument that the > original patch is correct), that would be great. Updated the patch with Austin's suggestion. I'm not personally ready to forbid the use of '..', although I certainly appreciate the motivation to keep our tests working in --root mode. test/README|9 + test/basic | 10 +- test/crypto|2 +- test/emacs |4 ++-- test/symbol-hiding |4 ++-- test/test-lib.sh | 18 +- 6 files changed, 28 insertions(+), 19 deletions(-) diff --git a/test/README b/test/README index be75e0e..8fbf78d 100644 --- a/test/README +++ b/test/README @@ -41,6 +41,15 @@ The following command-line options are available when running tests: As the names depend on the tests' file names, it is safe to run the tests with this option in parallel. +--root=:: + This runs the testsuites specified under a seperate directory. + However, caution is advised, as not all tests are maintained + with this relocation in mind, so some tests may behave + differently. + + Pointing this argument at a tmpfs filesystem can improve the + speed of the test suite for some users. + When invoking the test suite via "make test" any of the above options can be specified as follows: diff --git a/test/basic b/test/basic index d6e8c10..33bf711 100755 --- a/test/basic +++ b/test/basic @@ -51,9 +51,9 @@ test_expect_code 2 'failure to clean up causes the test to fail' ' # Ensure that all tests are being run test_begin_subtest 'Ensure that all available tests will be run by notmuch-test' -eval $(sed -n -e '/^TESTS="$/,/^"$/p' notmuch-test ../notmuch-test) +eval $(sed -n -e '/^TESTS="$/,/^"$/p' notmuch-test $TEST_DIRECTORY/notmuch-test) tests_in_suite=$(for i in $TESTS; do echo $i; done | sort) -available=$(ls -1 ../ | \ +available=$(ls -1 $TEST_DIRECTORY/ | \ sed -r -e "/^(aggregate-results.sh|Makefile|Makefile.local|notmuch-test)/d" \ -e "/^(README|test-lib.sh|test-lib.el|test-results|tmp.*|valgrind|corpus*)/d" \ -e "/^(emacs.expected-output|smtp-dummy|smtp-dummy.c|test-verbose|symbol-test.cc)/d" \ @@ -63,19 +63,19 @@ available=$(ls -1 ../ | \ | sort) test_expect_equal "$tests_in_suite" "$available" -EXPECTED=../test.expected-output +EXPECTED=$TEST_DIRECTORY/test.expected-output suppress_diff_date() { sed -e 's/\(.*\-\-\- test-verbose\.4\.\expected\).*/\1/' \ -e 's/\(.*\+\+\+ test-verbose\.4\.\output\).*/\1/' } test_begin_subtest "Ensure that test output is suppressed unless the test fails" -output=$(cd ..; ./test-verbose 2>&1 | suppress_diff_date) +output=$(cd $TEST_DIRECTORY; ./test-verbose 2>&1 | suppress_diff_date) expected=$(cat $EXPECTED/test-verbose-no | suppress_diff_date) test_expect_equal "$output" "$expected" test_begin_subtest "Ensure that -v does not suppress test output" -output=$(cd ..; ./test-verbose -v 2>&1 | suppress_diff_date) +output=$(cd $TEST_DIRECTORY; ./test-verbose -v 2>&1 | suppress_diff_date) expected=$(cat $EXPECTED/test-verbose-yes | suppress_diff_date) # Do not include the results of test-verbose in totals rm $TEST_DIRECTORY/test-results/test-verbose-* diff --git a/test/crypto b/test/crypto index 01daffe..7eb3559 100755 --- a/test/crypto +++ b/test/crypto @@ -12,7 +12,7 @@ add_gnupg_home () local output [ -d ${GNUPGHOME} ] && return mkdir -m 0700 "$GNUPGHOME" -gpg --no-tty --import <../gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 +gpg --no-tty --import <$TEST_DIRECTORY/gnupg-secret-key.asc >"$GNUPGHOME"/import.log 2>&1 test_debug "cat $GNUPGHOME/import.log" if (gpg --quick-random --version >/dev/null 2>&1) ; then echo quick-random >> "$GNUPGHOME"/gpg.conf diff --git a/test/emacs b/test/emacs index 6f82b08..f91078e 100755 --- a/test/emacs +++ b/test/emacs @@ -2,7 +2,7 @@ test_description="emacs interface" . test-lib.sh -EXPECTED=../emacs.expected-output +EXPECTED=$TEST_DIRECTORY/emacs.expected-output add_email_corpus @@ -81,7 +81,7 @@ mkdir -p mail/sent/cur mkdir -p mail/sent/new mkdir -p mail/sent/tmp -../smtp-dummy sent_message & +$TEST_DIRECTORY/smtp-dummy sent_message & smtp_dummy_pid=$! test_emacs "(setq message-send-mail-function 'message-smtpmail-send-it) (setq smtpmail-smtp-server \"localhost\") (setq smtpmail-smtp-service \"25025\") (notmuch-hello) (notmuch-mua-mail) (message-goto-to) (insert \"u...@example.com\nDate: Fri, 29 Mar 1974 10:00:00 -\") (message-goto-subject) (insert \"Testing message sent via SMTP\") (message-goto-body) (insert \"This is a test