Module Name: src
Committed By: pooka
Date: Wed Jun 16 15:17:37 UTC 2010
Modified Files:
src/external/bsd/atf/dist/atf-c: tcr.c tcr.h
src/external/bsd/atf/dist/atf-c++: formats.cpp tests.cpp tests.hpp
src/external/bsd/atf/dist/atf-report: atf-report.cpp
src/external/bsd/atf/dist/atf-run: atf-run.cpp
src/external/bsd/atf/dist/tests/atf/atf-c: t_macros.c
src/external/bsd/atf/dist/tests/atf/atf-report: t_integration.sh
Log Message:
Introduce expected failures to atf. They can be used to flag tests
which are known to fail, e.g.:
atf_tc_set_md_var(tc, "xfail", "PR kern/43456");
Expected failures do not count towards the ultimate pass/fail result
from the test run:
pain-rustique:39:~/<2>src/tests/fs/ptyfs> atf-run t_nullpts | atf-report
Tests root: /home/pooka/src/wholesrc2/src/tests/fs/ptyfs
t_nullpts (1/1): 1 test cases
nullrevoke: Expected failure: PR kern/43456
Summary for 1 test programs:
0 passed test cases.
0 failed test cases.
1 expected failures.
0 skipped test cases.
pain-rustique:40:~/<2>src/tests/fs/ptyfs> echo $?
0
However, an xfail test which passes will count as a failure, i.e.
xfail inverts test case success/fail. This way we can get a better
sense from the ultimate verdict of the NetBSD atf run by seeing if
there were any unexpected failures, i.e. new regressions.
This feature will be present in the upcoming atf 0.10 release,
possibly with finer grained control.
patch reviewed by jmmv
To generate a diff of this commit:
cvs rdiff -u -r1.1.1.4 -r1.2 src/external/bsd/atf/dist/atf-c/tcr.c
cvs rdiff -u -r1.1.1.3 -r1.2 src/external/bsd/atf/dist/atf-c/tcr.h
cvs rdiff -u -r1.1.1.3 -r1.2 src/external/bsd/atf/dist/atf-c++/formats.cpp
cvs rdiff -u -r1.1.1.4 -r1.2 src/external/bsd/atf/dist/atf-c++/tests.cpp \
src/external/bsd/atf/dist/atf-c++/tests.hpp
cvs rdiff -u -r1.1.1.1 -r1.2 \
src/external/bsd/atf/dist/atf-report/atf-report.cpp
cvs rdiff -u -r1.2 -r1.3 src/external/bsd/atf/dist/atf-run/atf-run.cpp
cvs rdiff -u -r1.1.1.4 -r1.2 \
src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c
cvs rdiff -u -r1.1.1.2 -r1.2 \
src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/external/bsd/atf/dist/atf-c/tcr.c
diff -u src/external/bsd/atf/dist/atf-c/tcr.c:1.1.1.4 src/external/bsd/atf/dist/atf-c/tcr.c:1.2
--- src/external/bsd/atf/dist/atf-c/tcr.c:1.1.1.4 Fri Jun 4 08:23:43 2010
+++ src/external/bsd/atf/dist/atf-c/tcr.c Wed Jun 16 15:17:37 2010
@@ -48,7 +48,8 @@
bool
state_allows_reason(atf_tcr_state_t state)
{
- return state == atf_tcr_failed_state || state == atf_tcr_skipped_state;
+ return state == atf_tcr_failed_state || state == atf_tcr_skipped_state
+ || state == atf_tcr_xfail_state;
}
static
@@ -106,6 +107,7 @@
const atf_tcr_state_t atf_tcr_passed_state = 0;
const atf_tcr_state_t atf_tcr_failed_state = 1;
const atf_tcr_state_t atf_tcr_skipped_state = 2;
+const atf_tcr_state_t atf_tcr_xfail_state = 3;
/*
* Constructors/destructors.
Index: src/external/bsd/atf/dist/atf-c/tcr.h
diff -u src/external/bsd/atf/dist/atf-c/tcr.h:1.1.1.3 src/external/bsd/atf/dist/atf-c/tcr.h:1.2
--- src/external/bsd/atf/dist/atf-c/tcr.h:1.1.1.3 Fri Jun 4 08:23:42 2010
+++ src/external/bsd/atf/dist/atf-c/tcr.h Wed Jun 16 15:17:37 2010
@@ -54,6 +54,7 @@
extern const atf_tcr_state_t atf_tcr_passed_state;
extern const atf_tcr_state_t atf_tcr_failed_state;
extern const atf_tcr_state_t atf_tcr_skipped_state;
+extern const atf_tcr_state_t atf_tcr_xfail_state;
/* Constructors/destructors. */
atf_error_t atf_tcr_init(atf_tcr_t *, int);
Index: src/external/bsd/atf/dist/atf-c++/formats.cpp
diff -u src/external/bsd/atf/dist/atf-c++/formats.cpp:1.1.1.3 src/external/bsd/atf/dist/atf-c++/formats.cpp:1.2
--- src/external/bsd/atf/dist/atf-c++/formats.cpp:1.1.1.3 Fri Jun 4 08:23:44 2010
+++ src/external/bsd/atf/dist/atf-c++/formats.cpp Wed Jun 16 15:17:37 2010
@@ -463,6 +463,7 @@
static const atf::parser::token_type& passed_type = 12;
static const atf::parser::token_type& failed_type = 13;
static const atf::parser::token_type& skipped_type = 14;
+static const atf::parser::token_type& xfail_type = 15;
static const atf::parser::token_type& info_type = 16;
class tokenizer : public atf::parser::tokenizer< std::istream > {
@@ -483,6 +484,7 @@
add_keyword("passed", passed_type);
add_keyword("failed", failed_type);
add_keyword("skipped", skipped_type);
+ add_keyword("xfail", xfail_type);
add_keyword("info", info_type);
}
};
@@ -860,6 +862,9 @@
throw parse_error(lineno, "The use.fs property requires a boolean"
" value");
}
+ } else if (name == "xfail") {
+ if (value.empty())
+ throw parse_error(lineno, "'xfail' requires a non-empty reason");
} else if (name.size() > 2 && name[0] == 'X' && name[1] == '-') {
// Any non-empty value is valid.
} else {
@@ -1155,8 +1160,8 @@
t = p.expect(comma_type, "`,'");
- t = p.expect(passed_type, failed_type, skipped_type,
- "passed, failed or skipped");
+ t = p.expect(passed_type, failed_type, skipped_type, xfail_type,
+ "passed, failed, skipped or xfail");
if (t.type() == passed_type) {
CALLBACK(p, got_tc_end(tcr(tcr::passed_state)));
} else if (t.type() == failed_type) {
@@ -1173,6 +1178,13 @@
throw parse_error(t.lineno(),
"Empty reason for skipped test case result");
CALLBACK(p, got_tc_end(tcr(tcr::skipped_state, reason)));
+ } else if (t.type() == xfail_type) {
+ t = p.expect(comma_type, "`,'");
+ std::string reason = text::trim(p.rest_of_line());
+ if (reason.empty())
+ throw parse_error(t.lineno(),
+ "Empty reason for xfail test case result");
+ CALLBACK(p, got_tc_end(tcr(tcr::xfail_state, reason)));
} else
UNREACHABLE;
@@ -1307,6 +1319,8 @@
str += "skipped, " + tcr.get_reason();
else if (tcr.get_state() == atf::tests::tcr::failed_state)
str += "failed, " + tcr.get_reason();
+ else if (tcr.get_state() == atf::tests::tcr::xfail_state)
+ str += "xfail, " + tcr.get_reason();
else
UNREACHABLE;
m_os << str << std::endl;
Index: src/external/bsd/atf/dist/atf-c++/tests.cpp
diff -u src/external/bsd/atf/dist/atf-c++/tests.cpp:1.1.1.4 src/external/bsd/atf/dist/atf-c++/tests.cpp:1.2
--- src/external/bsd/atf/dist/atf-c++/tests.cpp:1.1.1.4 Fri Jun 4 08:23:44 2010
+++ src/external/bsd/atf/dist/atf-c++/tests.cpp Wed Jun 16 15:17:37 2010
@@ -75,6 +75,7 @@
const impl::tcr::state impl::tcr::passed_state = atf_tcr_passed_state;
const impl::tcr::state impl::tcr::failed_state = atf_tcr_failed_state;
const impl::tcr::state impl::tcr::skipped_state = atf_tcr_skipped_state;
+const impl::tcr::state impl::tcr::xfail_state = atf_tcr_xfail_state;
impl::tcr::tcr(state s)
{
@@ -87,7 +88,7 @@
impl::tcr::tcr(state s, const std::string& r)
{
- PRE(s == failed_state || s == skipped_state);
+ PRE(s == failed_state || s == skipped_state || s == xfail_state);
PRE(!r.empty());
atf_error_t err = atf_tcr_init_reason_fmt(&m_tcr, s, "%s", r.c_str());
Index: src/external/bsd/atf/dist/atf-c++/tests.hpp
diff -u src/external/bsd/atf/dist/atf-c++/tests.hpp:1.1.1.4 src/external/bsd/atf/dist/atf-c++/tests.hpp:1.2
--- src/external/bsd/atf/dist/atf-c++/tests.hpp:1.1.1.4 Fri Jun 4 08:23:43 2010
+++ src/external/bsd/atf/dist/atf-c++/tests.hpp Wed Jun 16 15:17:37 2010
@@ -74,6 +74,7 @@
static const state passed_state;
static const state failed_state;
static const state skipped_state;
+ static const state xfail_state;
tcr(state);
tcr(state, const std::string&);
Index: src/external/bsd/atf/dist/atf-report/atf-report.cpp
diff -u src/external/bsd/atf/dist/atf-report/atf-report.cpp:1.1.1.1 src/external/bsd/atf/dist/atf-report/atf-report.cpp:1.2
--- src/external/bsd/atf/dist/atf-report/atf-report.cpp:1.1.1.1 Sat May 8 08:05:22 2010
+++ src/external/bsd/atf/dist/atf-report/atf-report.cpp Wed Jun 16 15:17:37 2010
@@ -176,7 +176,7 @@
ostream_ptr m_os;
size_t m_curtp, m_ntps;
- size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped;
+ size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped, m_tcs_xfail;
std::string m_tcname, m_tpname;
std::vector< std::string > m_failed_tcs;
std::vector< std::string > m_failed_tps;
@@ -197,6 +197,7 @@
m_tcs_passed = 0;
m_tcs_failed = 0;
m_tcs_skipped = 0;
+ m_tcs_xfail = 0;
m_ntps = ntps;
}
@@ -261,6 +262,9 @@
} else if (s == atf::tests::tcr::skipped_state) {
str = "Skipped: " + tcr.get_reason();
m_tcs_skipped++;
+ } else if (s == atf::tests::tcr::xfail_state) {
+ str = "Expected failure: " + tcr.get_reason();
+ m_tcs_xfail++;
} else
UNREACHABLE;
@@ -306,6 +310,10 @@
" failed test cases.",
" ", false)
<< std::endl;
+ (*m_os) << format_text_with_tag(to_string(m_tcs_xfail) +
+ " expected failures.",
+ " ", false)
+ << std::endl;
(*m_os) << format_text_with_tag(to_string(m_tcs_skipped) +
" skipped test cases.",
" ", false)
@@ -334,7 +342,7 @@
ostream_ptr m_os;
size_t m_curtp, m_ntps;
- size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped;
+ size_t m_tcs_passed, m_tcs_failed, m_tcs_skipped, m_tcs_xfail;
std::string m_tcname, m_tpname;
std::vector< std::string > m_failed_tcs;
std::vector< std::string > m_failed_tps;
Index: src/external/bsd/atf/dist/atf-run/atf-run.cpp
diff -u src/external/bsd/atf/dist/atf-run/atf-run.cpp:1.2 src/external/bsd/atf/dist/atf-run/atf-run.cpp:1.3
--- src/external/bsd/atf/dist/atf-run/atf-run.cpp:1.2 Thu Jun 10 15:27:02 2010
+++ src/external/bsd/atf/dist/atf-run/atf-run.cpp Wed Jun 16 15:17:37 2010
@@ -98,7 +98,7 @@
const atf::fs::path&);
atf::tests::tcr get_tcr(const std::string&, const atf::process::status&,
- const atf::fs::path&) const;
+ const atf::fs::path&, const std::string&) const;
public:
atf_run(void);
@@ -214,13 +214,16 @@
atf::tests::tcr
atf_run::get_tcr(const std::string& broken_reason,
const atf::process::status& s,
- const atf::fs::path& resfile)
+ const atf::fs::path& resfile,
+ const std::string& config_xfail)
const
{
using atf::tests::tcr;
+ const bool default_fail = !config_xfail.empty();
- if (!broken_reason.empty())
+ if (!broken_reason.empty()) {
return tcr(tcr::failed_state, broken_reason);
+ }
if (s.exited()) {
try {
@@ -229,10 +232,15 @@
if (s.exitstatus() == EXIT_SUCCESS)
return tcr(tcr::failed_state, "Test case exited "
"successfully but reported failure");
+ if (default_fail)
+ return tcr(tcr::xfail_state, config_xfail);
} else {
if (s.exitstatus() != EXIT_SUCCESS)
return tcr(tcr::failed_state, "Test case exited "
"with error but reported success");
+ if (default_fail)
+ return tcr(tcr::failed_state, "Test case is expected to "
+ "fail but reported success");
}
return ret;
} catch (const atf::formats::format_error& e) {
@@ -248,10 +256,13 @@
std::string(e.what()));
}
} else if (s.signaled()) {
- return tcr(tcr::failed_state,
- "Test program received signal " +
- atf::text::to_string(s.termsig()) +
- (s.coredump() ? " (core dumped)" : ""));
+ if (default_fail)
+ return tcr(tcr::xfail_state, config_xfail);
+ else
+ return tcr(tcr::failed_state,
+ "Test program received signal " +
+ atf::text::to_string(s.termsig()) +
+ (s.coredump() ? " (core dumped)" : ""));
} else {
UNREACHABLE;
throw std::runtime_error("Unknown exit status");
@@ -310,6 +321,11 @@
continue;
}
+ std::string xfail_reason = "";
+ if (tcmd.find("xfail") != tcmd.end()) {
+ xfail_reason = (*tcmd.find("xfail")).second;
+ }
+
const atf::fs::path resfile = resdir.get_path() / "tcr";
INV(!atf::fs::exists(resfile));
try {
@@ -330,7 +346,7 @@
// TODO: Force deletion of workdir.
- tcr = get_tcr(s.first, s.second, resfile);
+ tcr = get_tcr(s.first, s.second, resfile, xfail_reason);
} else {
std::pair< std::string, const atf::process::status > s =
impl::run_test_case(tp, tcname, "body", tcmd, config,
@@ -338,7 +354,7 @@
(void)impl::run_test_case(tp, tcname, "cleanup", tcmd, config,
resfile, ro_workdir, w);
- tcr = get_tcr(s.first, s.second, resfile);
+ tcr = get_tcr(s.first, s.second, resfile, xfail_reason);
}
w.end_tc(tcr);
if (tcr.get_state() == atf::tests::tcr::failed_state)
Index: src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c
diff -u src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c:1.1.1.4 src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c:1.2
--- src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c:1.1.1.4 Fri Jun 4 08:23:48 2010
+++ src/external/bsd/atf/dist/tests/atf/atf-c/t_macros.c Wed Jun 16 15:17:37 2010
@@ -589,6 +589,7 @@
atf_tc_set_md_var(tc, "descr", "Tests that format strings passed "
"as part of the automatically-generated messages "
"do not get expanded");
+ atf_tc_set_md_var(tc, "xfail", "broken here, fixed in atf 0.10");
}
ATF_TC_BODY(msg_embedded_fmt, tc)
{
@@ -609,9 +610,6 @@
{ NULL, NULL, false, NULL }
};
- atf_tc_skip("Broken test. XXX: This should really be signaled as an "
- "expected failure, not as a skipped test.");
-
for (t = &tests[0]; t->head != NULL; t++) {
printf("Checking with an expected '%s' message\n", t->msg);
Index: src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh
diff -u src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh:1.1.1.2 src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh:1.2
--- src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh:1.1.1.2 Fri Jun 4 08:23:50 2010
+++ src/external/bsd/atf/dist/tests/atf/atf-report/t_integration.sh Wed Jun 16 15:17:37 2010
@@ -223,6 +223,7 @@
Summary for 5 test programs:
2 passed test cases.
2 failed test cases.
+ 0 expected failures.
0 skipped test cases.
EOF
