This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 992c7cc3f27 HDDS-14072. Show structured test summary in CI result
(#10857)
992c7cc3f27 is described below
commit 992c7cc3f27109c5c4ac56f159103a3cae6d7c8d
Author: Chi-Hsuan Huang <[email protected]>
AuthorDate: Tue Sep 22 00:50:25 2026 +0800
HDDS-14072. Show structured test summary in CI result (#10857)
Co-authored-by: Claude Code (Opus 4.8)
---
.github/workflows/check.yml | 8 ++
dev-support/ci/junit_summary.bats | 22 ++++
dev-support/ci/junit_summary.py | 189 ++++++++++++++++++++++++++
dev-support/ci/test_junit_summary.py | 226 ++++++++++++++++++++++++++++++++
hadoop-ozone/dev-support/checks/bats.sh | 1 +
5 files changed, 446 insertions(+)
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 1a5db91c279..43da8c72d15 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -266,6 +266,7 @@ jobs:
hadoop-ozone/dev-support/checks/_summary.sh
target/$SCRIPT/summary.txt
- name: Archive build results
+ id: archive
if: ${{ !cancelled() }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
# v7.0.1
with:
@@ -274,6 +275,13 @@ jobs:
path: target/${{ inputs.script }}
continue-on-error: true
+ - name: Test summary
+ if: ${{ !cancelled() }}
+ continue-on-error: true
+ run: python3 dev-support/ci/junit_summary.py >> "$GITHUB_STEP_SUMMARY"
+ env:
+ JUNIT_REPORT_URL: ${{ steps.archive.outputs.artifact-url }}
+
# The following steps are hard-coded to be run only for 'build' check,
# to avoid the need for 3 more inputs.
- name: Store binaries for tests
diff --git a/dev-support/ci/junit_summary.bats
b/dev-support/ci/junit_summary.bats
new file mode 100644
index 00000000000..4f6003c6013
--- /dev/null
+++ b/dev-support/ci/junit_summary.bats
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+@test "junit_summary.py unit tests" {
+ cd "${BATS_TEST_DIRNAME}"
+ run python3 -m unittest test_junit_summary
+ [ "$status" -eq 0 ]
+}
diff --git a/dev-support/ci/junit_summary.py b/dev-support/ci/junit_summary.py
new file mode 100755
index 00000000000..a14242fcdf3
--- /dev/null
+++ b/dev-support/ci/junit_summary.py
@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Parse Maven Surefire/Failsafe JUnit XML reports and print a Markdown test
summary.
+
+Adapted from Apache Kafka's .github/scripts/junit.py (summary format),
reworked for Maven Surefire XML.
+
+Intended for GitHub Actions: junit_summary.py [--path DIR] >>
"$GITHUB_STEP_SUMMARY"
+Optional env: JUNIT_REPORT_URL (link to the archived test report artifact).
+Prints nothing and exits 0 when no reports are found. Always exits 0.
+"""
+
+import argparse
+import dataclasses
+import html
+import os
+import sys
+import xml.etree.ElementTree as ET
+from glob import glob
+
+PASSED = "✅ PASSED"
+FAILED = "❌ FAILED"
+FLAKY = "⚠️ FLAKY"
+SKIPPED = "🙈 SKIPPED"
+
+FAIL_TAGS = frozenset(("failure", "error"))
+FLAKY_TAGS = frozenset(("flakyFailure", "flakyError"))
+MESSAGE_LIMIT = 300
+
+
[email protected]
+class TestCase:
+ module: str
+ class_name: str
+ test_name: str
+ time: float
+ status: str # passed | failed | flaky | skipped
+ message: str = ""
+
+
+def module_name(xml_path):
+ # Works for both layouts: <module>/target/surefire-reports/TEST-*.xml
(normal run)
+ # and target/<check>/<module-path>/TEST-*.xml (failed tests moved by
_mvn_unit_report.sh).
+ parts = os.path.normpath(xml_path).split(os.sep)[:-1]
+ while parts and parts[-1] in ("surefire-reports", "failsafe-reports",
"target"):
+ parts.pop()
+ return parts[-1] if parts else "-"
+
+
+def clean_message(elem):
+ message = " ".join((elem.get("message") or (elem.text or "")).split())
+ if len(message) > MESSAGE_LIMIT:
+ # JUnit 5 appends "==> expected: ... but was: ..." after the custom
message; when a test embeds
+ # a whole log as the custom message, drop it and keep the tail with the
actual failure
+ idx = message.find(" ==> expected:")
+ if idx != -1:
+ message = message[idx + len(" ==> "):]
+ if len(message) > MESSAGE_LIMIT:
+ message = message[:MESSAGE_LIMIT] + "..."
+ return message
+
+
+def parse_report(xml_path):
+ module = module_name(xml_path)
+ cases = []
+ for testcase in ET.parse(xml_path).getroot().iter("testcase"):
+ failure = next((c for c in testcase if c.tag in FAIL_TAGS), None)
+ flaky = next((c for c in testcase if c.tag in FLAKY_TAGS), None)
+ skipped = next((c for c in testcase if c.tag == "skipped"), None)
+ status, message = "passed", ""
+ if failure is not None:
+ status, message = "failed", clean_message(failure)
+ elif flaky is not None:
+ status, message = "flaky", clean_message(flaky)
+ elif skipped is not None:
+ status = "skipped"
+ cases.append(TestCase(module, testcase.get("classname") or "",
testcase.get("name") or "",
+ float(testcase.get("time") or 0), status, message))
+ return cases
+
+
+def format_time(seconds):
+ minutes, secs = divmod(int(seconds), 60)
+ hours, minutes = divmod(minutes, 60)
+ if hours:
+ return "%dh%dm%ds" % (hours, minutes, secs)
+ if minutes:
+ return "%dm%ds" % (minutes, secs)
+ return "%ds" % secs
+
+
+def cell(text):
+ return html.escape(" ".join(str(text).split())).replace("|", "\\|")
+
+
+def render_table(title, header, rows):
+ lines = ["<details><summary><b>%s (%d)</b></summary>" % (title, len(rows)),
""]
+ lines.append("|" + "|".join(header) + "|")
+ lines.append("|" + "|".join("---" for _ in header) + "|")
+ lines.extend("|" + "|".join(cell(value) for value in row) + "|" for row in
rows)
+ lines.extend(["", "</details>", ""])
+ return lines
+
+
+def render_summary(cases):
+ def select(status):
+ return [c for c in cases if c.status == status]
+
+ def full_name(case):
+ return "%s.%s" % (case.class_name, case.test_name)
+
+ passed, failed, flaky, skipped = select("passed"), select("failed"),
select("flaky"), select("skipped")
+ lines = ["## Test Summary", ""]
+ # sum of per-test times, not wall clock (parallel forks make wall clock much
shorter)
+ lines.append("%d tests run in %s (total test time):" % (len(cases),
format_time(sum(c.time for c in cases))))
+ for group, label in ((passed, PASSED), (failed, FAILED), (flaky, FLAKY),
(skipped, SKIPPED)):
+ if group:
+ emoji, word = label.split(" ", 1)
+ lines.append("- %s %d %s" % (emoji, len(group), word))
+ lines.append("")
+ report_url = os.environ.get("JUNIT_REPORT_URL")
+ if report_url:
+ lines.extend(["[Download test artifacts](%s)" % report_url, ""])
+ if failed:
+ lines.extend(render_table(FAILED, ["Module", "Test", "Message", "Time"],
+ [[c.module, full_name(c), c.message,
format_time(c.time)] for c in failed]))
+ if flaky:
+ lines.extend(render_table(FLAKY, ["Module", "Test", "Message", "Time"],
+ [[c.module, full_name(c), c.message,
format_time(c.time)] for c in flaky]))
+ if skipped:
+ lines.extend(render_table(SKIPPED, ["Module", "Test"],
+ [[c.module, full_name(c)] for c in skipped]))
+ return "\n".join(lines) + "\n"
+
+
+REPORT_PATTERNS = (
+ os.path.join("**", "surefire-reports", "TEST-*.xml"),
+ os.path.join("**", "failsafe-reports", "TEST-*.xml"),
+ os.path.join("target", "*", "**", "TEST-*.xml"),
+)
+
+
+def find_reports(base):
+ found = set()
+ for pattern in REPORT_PATTERNS:
+ found.update(glob(os.path.join(base, pattern), recursive=True))
+ # junit.sh keeps per-iteration copies under target/<check>/iterationN/ when
ITERATIONS>1;
+ # exclude them like _mvn_unit_report.sh does, so reruns are not counted
multiple times.
+ return sorted(path for path in found if "/iteration" not in
path.replace(os.sep, "/"))
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description="Print a Markdown summary of
JUnit XML test reports.")
+ parser.add_argument("--path", default=".", help="directory to scan for
TEST-*.xml reports")
+ try:
+ args = parser.parse_args(argv)
+ except SystemExit:
+ return 0
+ # This script only decorates the step summary; it must NEVER fail the check,
so exit 0 no matter what.
+ try:
+ cases = []
+ for report in find_reports(args.path):
+ try:
+ cases.extend(parse_report(report))
+ except Exception as e:
+ print("Skipping unreadable report %s: %s" % (report, e),
file=sys.stderr)
+ if cases:
+ print(render_summary(cases), end="")
+ except Exception as e:
+ print("junit_summary failed: %s" % e, file=sys.stderr)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/dev-support/ci/test_junit_summary.py
b/dev-support/ci/test_junit_summary.py
new file mode 100644
index 00000000000..62d58b6c0d5
--- /dev/null
+++ b/dev-support/ci/test_junit_summary.py
@@ -0,0 +1,226 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for junit_summary.py. Run: cd dev-support/ci && python3 -m unittest
test_junit_summary -v"""
+
+import contextlib
+import io
+import os
+import tempfile
+import unittest
+
+import junit_summary
+
+SAMPLE_XML = """<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="org.apache.hadoop.ozone.TestExample" tests="4" failures="1"
errors="0" skipped="1" time="12.5">
+ <testcase name="testPasses" classname="org.apache.hadoop.ozone.TestExample"
time="1.5"/>
+ <testcase name="testFails" classname="org.apache.hadoop.ozone.TestExample"
time="2.0">
+ <failure message="expected: <1> but was: |2|"
type="AssertionError">stack trace here</failure>
+ </testcase>
+ <testcase name="testFlaky" classname="org.apache.hadoop.ozone.TestExample"
time="3.0">
+ <flakyFailure message="Connection refused"
type="IOException">stack</flakyFailure>
+ </testcase>
+ <testcase name="testSkipped" classname="org.apache.hadoop.ozone.TestExample"
time="0.0">
+ <skipped message="disabled"/>
+ </testcase>
+</testsuite>
+"""
+
+
+class TestParseReport(unittest.TestCase):
+
+ def write_report(self, tmp, relpath, content=SAMPLE_XML):
+ path = os.path.join(tmp, relpath)
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as f:
+ f.write(content)
+ return path
+
+ def test_classification(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = self.write_report(tmp,
"hadoop-hdds/common/target/surefire-reports/TEST-x.xml")
+ cases = junit_summary.parse_report(path)
+ by_name = {c.test_name: c for c in cases}
+ self.assertEqual(len(cases), 4)
+ self.assertEqual(by_name["testPasses"].status, "passed")
+ self.assertEqual(by_name["testFails"].status, "failed")
+ self.assertEqual(by_name["testFlaky"].status, "flaky")
+ self.assertEqual(by_name["testSkipped"].status, "skipped")
+ self.assertEqual(by_name["testFails"].class_name,
"org.apache.hadoop.ozone.TestExample")
+ self.assertAlmostEqual(by_name["testFlaky"].time, 3.0)
+
+ def test_message_is_collapsed_raw_text(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ path = self.write_report(tmp, "m/target/surefire-reports/TEST-x.xml")
+ cases = junit_summary.parse_report(path)
+ failed = next(c for c in cases if c.status == "failed")
+ self.assertEqual(failed.message, "expected: <1> but was: |2|")
+ flaky = next(c for c in cases if c.status == "flaky")
+ self.assertEqual(flaky.message, "Connection refused")
+
+ def test_message_keeps_junit5_tail_when_custom_message_floods(self):
+ # assertEquals(expected, actual, message) renders as "<custom message> ==>
expected: ... but was: ...";
+ # some tests embed whole logs as the custom message, which would push the
tail past the truncation limit
+ flood = "log line " * 100
+ xml = ('<testsuite><testcase name="t" classname="C" time="1">'
+ '<failure message="client log: %s ==> expected: <1> but
was: <2>"/>'
+ '</testcase></testsuite>' % flood)
+ with tempfile.TemporaryDirectory() as tmp:
+ path = self.write_report(tmp, "m/target/surefire-reports/TEST-x.xml",
content=xml)
+ cases = junit_summary.parse_report(path)
+ self.assertEqual(cases[0].message, "expected: <1> but was: <2>")
+
+ def test_message_junit5_tail_edge_cases(self):
+ fromstring = junit_summary.ET.fromstring
+ # short custom messages are meaningful and kept in full
+ elem = fromstring('<failure message="wrong count ==> expected:
<1> but was: <2>"/>')
+ self.assertEqual(junit_summary.clean_message(elem), "wrong count ==>
expected: <1> but was: <2>")
+ # assertion values containing " ==> " must not corrupt the tail
+ long_context = "x" * 400
+ elem = fromstring('<failure message="%s ==> expected: <a ==>
b> but was: <c>"/>' % long_context)
+ self.assertEqual(junit_summary.clean_message(elem), "expected: <a ==> b>
but was: <c>")
+ # long messages with " ==> " but no assertion framing fall back to head
truncation
+ elem = fromstring('<failure message="pipeline A ==> B failed %s"/>' %
long_context)
+ self.assertTrue(junit_summary.clean_message(elem).startswith("pipeline A
==> B failed"))
+
+ def test_module_name_from_path(self):
+ # normal surefire layout: module is the dir above target/
+
self.assertEqual(junit_summary.module_name("hadoop-hdds/common/target/surefire-reports/TEST-a.xml"),
"common")
+ # layout after _mvn_unit_report.sh moves a failed test's XML under
target/<check>/<module-path>/
+
self.assertEqual(junit_summary.module_name("target/unit/hadoop-hdds/common/TEST-a.xml"),
"common")
+ self.assertEqual(junit_summary.module_name("TEST-a.xml"), "-")
+
+
+class TestRender(unittest.TestCase):
+
+ def make_cases(self):
+ tc = junit_summary.TestCase
+ return [
+ tc("common", "org.X", "ok", 1.0, "passed"),
+ tc("common", "org.X", "bad", 2.0, "failed", "boom"),
+ tc("common", "org.X", "shaky", 3.0, "flaky", "flap"),
+ tc("common", "org.X", "skip", 0.0, "skipped"),
+ ]
+
+ def test_format_time(self):
+ self.assertEqual(junit_summary.format_time(59), "59s")
+ self.assertEqual(junit_summary.format_time(779), "12m59s")
+ self.assertEqual(junit_summary.format_time(3725), "1h2m5s")
+
+ def test_render_counts_list(self):
+ md = junit_summary.render_summary(self.make_cases())
+ self.assertIn("## Test Summary", md)
+ self.assertIn("4 tests run in 6s (total test time):", md)
+ self.assertIn("- ✅ 1 PASSED\n- ❌ 1 FAILED\n- ⚠️ 1 FLAKY\n- 🙈 1 SKIPPED",
md)
+
+ def test_render_counts_omit_zero(self):
+ cases = [c for c in self.make_cases() if c.status == "passed"]
+ md = junit_summary.render_summary(cases)
+ self.assertIn("- ✅ 1 PASSED", md)
+ self.assertNotIn("FAILED", md)
+ self.assertNotIn("FLAKY", md)
+ self.assertNotIn("SKIPPED", md)
+
+ def test_render_tables(self):
+ md = junit_summary.render_summary(self.make_cases())
+ self.assertIn("❌ FAILED (1)", md)
+ self.assertIn("|common|org.X.bad|boom|2s|", md)
+ self.assertIn("⚠️ FLAKY (1)", md)
+ self.assertIn("|common|org.X.shaky|flap|3s|", md)
+ self.assertIn("🙈 SKIPPED (1)", md)
+
+ def test_render_escapes_every_cell(self):
+ # parameterized test names can contain pipes and HTML-significant chars
+ cases = [junit_summary.TestCase("common", "org.X", "test[a|b]<c>", 1.0,
"failed", "got |x| <y>")]
+ md = junit_summary.render_summary(cases)
+ self.assertIn("|common|org.X.test[a\\|b]<c>|got \\|x\\|
<y>|1s|", md)
+
+ def test_render_report_url(self):
+ os.environ["JUNIT_REPORT_URL"] = "https://example.com/artifact";
+ try:
+ md = junit_summary.render_summary(self.make_cases())
+ finally:
+ del os.environ["JUNIT_REPORT_URL"]
+ self.assertIn("[Download test artifacts](https://example.com/artifact)",
md)
+
+
+class TestMain(unittest.TestCase):
+
+ write_report = TestParseReport.write_report
+
+ def run_main(self, argv):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ code = junit_summary.main(argv)
+ return code, out.getvalue()
+
+ def test_find_reports_dedupes_both_trees(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ self.write_report(tmp,
"hadoop-hdds/common/target/surefire-reports/TEST-a.xml")
+ self.write_report(tmp, "target/unit/surefire-reports/TEST-b.xml")
+ reports = junit_summary.find_reports(tmp)
+ self.assertEqual(len(reports), 2)
+
+ def test_find_reports_skips_iteration_dirs(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ self.write_report(tmp, "target/unit/TEST-a.xml")
+ self.write_report(tmp, "target/unit/iteration2/TEST-a.xml")
+ reports = junit_summary.find_reports(tmp)
+ self.assertEqual(len(reports), 1)
+
+ def test_main_no_reports_is_silent_noop(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ code, out = self.run_main(["--path", tmp])
+ self.assertEqual(code, 0)
+ self.assertEqual(out, "")
+
+ def test_main_prints_summary(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ self.write_report(tmp, "m/target/surefire-reports/TEST-a.xml")
+ code, out = self.run_main(["--path", tmp])
+ self.assertEqual(code, 0)
+ self.assertIn("## Test Summary", out)
+ self.assertIn("4 tests run", out)
+
+ def test_main_survives_malformed_xml(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ self.write_report(tmp, "m/target/surefire-reports/TEST-a.xml")
+ self.write_report(tmp, "m/target/surefire-reports/TEST-bad.xml",
content="<not-xml")
+ with contextlib.redirect_stderr(io.StringIO()):
+ code, out = self.run_main(["--path", tmp])
+ self.assertEqual(code, 0)
+ self.assertIn("4 tests run", out)
+
+ def test_main_survives_non_parse_errors(self):
+ # a ValueError (bad time attribute), not an ET.ParseError, must not break
the exit-0 guarantee
+ bad_time = '<testsuite><testcase name="t" classname="C"
time="not-a-number"/></testsuite>'
+ with tempfile.TemporaryDirectory() as tmp:
+ self.write_report(tmp, "m/target/surefire-reports/TEST-a.xml")
+ self.write_report(tmp, "m/target/surefire-reports/TEST-badtime.xml",
content=bad_time)
+ with contextlib.redirect_stderr(io.StringIO()):
+ code, out = self.run_main(["--path", tmp])
+ self.assertEqual(code, 0)
+ self.assertIn("4 tests run", out)
+
+ def test_main_bad_args_still_exit_zero(self):
+ with contextlib.redirect_stderr(io.StringIO()):
+ code = junit_summary.main(["--no-such-flag"])
+ self.assertEqual(code, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/hadoop-ozone/dev-support/checks/bats.sh
b/hadoop-ozone/dev-support/checks/bats.sh
index caffc8f2e2a..839b4dd11e6 100755
--- a/hadoop-ozone/dev-support/checks/bats.sh
+++ b/hadoop-ozone/dev-support/checks/bats.sh
@@ -36,6 +36,7 @@ find * \( \
-or -path dev-support/ci/selective_ci_checks.bats \
-or -path dev-support/ci/pr_title_check.bats \
-or -path dev-support/ci/find_test_class_project.bats \
+ -or -path dev-support/ci/junit_summary.bats \
\) -print0 \
| xargs -0 -n1 bats --formatter tap \
| tee -a "${REPORT_DIR}/output.log"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
