chia7712 commented on code in PR #17034:
URL: https://github.com/apache/kafka/pull/17034#discussion_r1738517170
##########
.github/scripts/junit.py:
##########
@@ -148,29 +168,73 @@ def pretty_time_duration(seconds: float) -> str:
total_failures += suite.failures
total_errors += suite.errors
total_time += suite.time
- for test_failure in suite.test_failures:
+
+ # Due to how the Develocity Test Retry plugin interacts with
our geneated ClusterTests, we can see
+ # tests pass and then fail in the same run. Because of this,
we need to capture all passed and all
+ # failed for each suite. Then we can find flakes by taking the
intersection of those two.
+ all_suite_passed = {test.key() for test in suite.passed_tests}
+ all_suite_failed = {test.key() for test in suite.failed_tests}
+ flaky = all_suite_passed & all_suite_failed
+ total_flaky += len(flaky)
+
+ # Display failures first
+ for test_failure in suite.failed_tests:
+ if test_failure.key() in flaky:
+ continue
logger.debug(f"Found test failure: {test_failure}")
simple_class_name = test_failure.class_name.split(".")[-1]
- table.append(("❌", simple_class_name,
test_failure.test_name, test_failure.failure_message,
f"{test_failure.time:0.2f}s"))
+ failed_table.append((simple_class_name,
test_failure.test_name, test_failure.failure_message,
f"{test_failure.time:0.2f}s"))
+ for test_failure in suite.failed_tests:
+ if test_failure.key() not in flaky:
+ continue
+ logger.debug(f"Found flaky test: {test_failure}")
+ simple_class_name = test_failure.class_name.split(".")[-1]
+ flaky_table.append((simple_class_name,
test_failure.test_name, test_failure.failure_message,
f"{test_failure.time:0.2f}s"))
for skipped_test in suite.skipped_tests:
simple_class_name = skipped_test.class_name.split(".")[-1]
logger.debug(f"Found skipped test: {skipped_test}")
- table.append(("⚠️", simple_class_name,
skipped_test.test_name, "Skipped", ""))
+ skipped_table.append((simple_class_name,
skipped_test.test_name))
duration = pretty_time_duration(total_time)
+ logger.info(f"Finished processing {len(reports)} reports")
# Print summary
report_url = get_env("REPORT_URL")
report_md = f"Download [HTML report]({report_url})."
- summary = f"{total_tests} tests run in {duration}, {total_failures} failed
❌, {total_skipped} skipped ⚠️, {total_errors} errors."
- logger.debug(summary)
+ summary = f"{total_tests} tests run in {duration}, {total_failures}
{FAILED}, {total_flaky} {FLAKY}, {total_skipped} {SKIPPED}, and {total_errors}
errors."
Review Comment:
The tests shown by "Failed Tests" table exclude the "flaky". Does it confuse
readers that the "number of failed" is NOT equal to the number of tests shown
by "Failed Tests" table.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
