The libhugetlbfs test suite has various tests for which a failure (under
certain circumstances) is expected and not indicative of an error. It would be
desirable to specially annotate these cases to eliminate the need for the
tester to know which failures are known and which could be bugs.
This patch creates some test harness infrastructure that can be used to achieve
this goal. How does it work? If a test case needs expected failure
validation, a shell script (named <testcase>.sh) is placed in the tests/
directory. The test invocation in run_tests.sh is changed to execute the shell
script instead of the c program directly. The shell script must perform any
required discovery, execute the real test case, and interpret the result.
Comments?
Changes since V1:
- Revamped the build so wrappers properly depend on their test cases
Signed-off-by: Adam Litke <[EMAIL PROTECTED]>
---
tests/Makefile | 19 ++++++++++++++++++-
tests/hugetests.h | 2 ++
tests/run_tests.sh | 8 ++++++++
3 files changed, 28 insertions(+), 1 deletions(-)
diff --git a/tests/Makefile b/tests/Makefile
index 27e5dd5..a7cb328 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -16,6 +16,7 @@ LDSCRIPT_TESTS = zero_filesize_segment
HUGELINK_TESTS = linkhuge linkhuge_nofd linkshare
HUGELINK_RW_TESTS = linkhuge_rw
STRESS_TESTS = mmap-gettest mmap-cow shm-gettest shm-getraw shm-fork
+WRAPPERS =
HELPERS = get_hugetlbfs_path
HELPER_LIBS = libheapshrink.so
BADTOOLCHAIN = bad-toolchain.sh
@@ -66,7 +67,13 @@ ifdef CC64
ALLTESTS += $(TESTS_64:%=obj64/%)
endif
-all: $(ALLTESTS) $(ALLHELPERS) $(ALLHELPERLIBS)
+objs_needing_wrappers = \
+ $(foreach W,$(WRAPPERS:%.sh=%),$(filter $(1)/$(W),$(ALLTESTS)))
+WRAPPERS32 = $(addsuffix .sh,$(call objs_needing_wrappers,obj32))
+WRAPPERS64 = $(addsuffix .sh,$(call objs_needing_wrappers,obj64))
+ALLWRAPPERS = $(WRAPPERS32) $(WRAPPERS64)
+
+all: $(ALLTESTS) $(ALLHELPERS) $(ALLHELPERLIBS) $(ALLWRAPPERS)
shmoverride_linked.c: shmoverride_unlinked.c
ln -s shmoverride_unlinked.c shmoverride_linked.c
@@ -185,6 +192,14 @@ $(HELPERS:%=obj64/%): %: %.o
@$(VECHO) LD64 "(helper)" $@
$(CC64) $(LDFLAGS) $(LDFLAGS64) -o $@ $^ $(LDLIBS) -lhugetlbfs
+$(WRAPPERS32): obj32/%.sh: %.sh obj32/%
+ @$(VECHO) COPY "(wrapped test)" $@
+ @cp -f $< $@
+
+$(WRAPPERS64): obj64/%.sh: %.sh obj64/%
+ @$(VECHO) COPY "(wrapped test)" $@
+ @cp -f $< $@
+
clean:
@$(VECHO) CLEAN "(tests)"
rm -f *~ *.o *.so *.a *.d core a.out
@@ -201,6 +216,7 @@ obj32/install:
$(INSTALL) -d $(DESTDIR)$(INST_TESTSDIR32)
$(INSTALL) -d $(DESTDIR)$(INST_TESTSDIR32)/obj32
$(INSTALL) -m 755 $(TESTS:%=obj32/%) $(DESTDIR)$(INST_TESTSDIR32)/obj32
+ $(INSTALL) -m 755 $(WRAPPERS32) $(DESTDIR)$(INST_TESTSDIR32)/obj32
$(INSTALL) -m 755 $(HELPERS:%=obj32/%)
$(DESTDIR)$(INST_TESTSDIR32)/obj32
$(INSTALL) -m 755 $(HELPER_LIBS:%=obj32/%)
$(DESTDIR)$(INST_TESTSDIR32)/obj32
$(INSTALL) -m 755 run_tests.sh $(DESTDIR)$(INST_TESTSDIR32)
@@ -210,6 +226,7 @@ obj64/install:
$(INSTALL) -d $(DESTDIR)$(INST_TESTSDIR64)
$(INSTALL) -d $(DESTDIR)$(INST_TESTSDIR64)/obj64
$(INSTALL) -m 755 $(TESTS:%=obj64/%) $(DESTDIR)$(INST_TESTSDIR64)/obj64
+ $(INSTALL) -m 755 $(WRAPPERS64) $(DESTDIR)$(INST_TESTSDIR64)/obj64
$(INSTALL) -m 755 $(HELPERS:%=obj64/%)
$(DESTDIR)$(INST_TESTSDIR64)/obj64
$(INSTALL) -m 755 $(HELPER_LIBS:%=obj64/%)
$(DESTDIR)$(INST_TESTSDIR64)/obj64
$(INSTALL) -m 755 $(TESTS_64:%=obj64/%)
$(DESTDIR)$(INST_TESTSDIR64)/obj64
diff --git a/tests/hugetests.h b/tests/hugetests.h
index 270923b..b860244 100644
--- a/tests/hugetests.h
+++ b/tests/hugetests.h
@@ -31,6 +31,8 @@
#define RC_PASS 0
#define RC_CONFIG 1
#define RC_FAIL 2
+#define RC_XFAIL 3 /* Expected Failure */
+#define RC_XPASS 4 /* Unexpected Pass */
#define RC_BUG 99
#define FOURGB (1UL << 32)
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 5b2a955..19cd9ee 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -19,6 +19,8 @@ for BITS in 32 64; do
tot_config[$BITS]=0
tot_signal[$BITS]=0
tot_strange[$BITS]=0
+ tot_xpass[$BITS]=0
+ tot_xfail[$BITS]=0
tot_skip[$BITS]=0
done
@@ -110,6 +112,10 @@ run_test_bits () {
tot_config[$BITS]=$[tot_config[$BITS] + 1]
elif [ "$rc" == "2" ]; then
tot_fail[$BITS]=$[tot_fail[$BITS] + 1]
+ elif [ "$rc" == "3" ]; then
+ tot_xfail[$BITS]=$[tot_xfail[$BITS] + 1]
+ elif [ "$rc" == "4" ]; then
+ tot_xpass[$BITS]=$[tot_xpass[$BITS] + 1]
elif [ "$rc" -gt 127 ]; then
tot_signal[$BITS]=$[tot_signal[$BITS] + 1]
else
@@ -421,5 +427,7 @@ echo -e "* PASS:
${tot_pass[32]}\t${tot_pass[64]}"
echo -e "* FAIL: ${tot_fail[32]}\t${tot_fail[64]}"
echo -e "* Killed by signal: ${tot_signal[32]}\t${tot_signal[64]}"
echo -e "* Bad configuration: ${tot_config[32]}\t${tot_config[64]}"
+echo -e "* Expected FAIL: ${tot_xfail[32]}\t${tot_xfail[64]}"
+echo -e "* Unexpected PASS: ${tot_xpass[32]}\t${tot_xpass[64]}"
echo -e "* Strange test result: ${tot_strange[32]}\t${tot_strange[64]}"
echo -e "**********"
-------------------------------------------------------------------------
This SF.Net email is sponsored by the Moblin Your Move Developer's challenge
Build the coolest Linux based applications with Moblin SDK & win great prizes
Grand prize is a trip for two to an Open Source event anywhere in the world
http://moblin-contest.org/redirect.php?banner_id=100&url=/
_______________________________________________
Libhugetlbfs-devel mailing list
Libhugetlbfs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/libhugetlbfs-devel
