Github user HyukjinKwon commented on a diff in the pull request:
https://github.com/apache/spark/pull/21107#discussion_r183230184
--- Diff: python/run-tests.py ---
@@ -109,8 +111,34 @@ def run_individual_python_test(test_name,
pyspark_python):
# this code is invoked from a thread other than the main
thread.
os._exit(-1)
else:
- per_test_output.close()
- LOGGER.info("Finished test(%s): %s (%is)", pyspark_python,
test_name, duration)
+ skipped_counts = 0
+ try:
+ per_test_output.seek(0)
+ # Here expects skipped test output from unittest when
verbosity level is
+ # 2 (or --verbose option is enabled).
+ decoded_lines = map(lambda line: line.decode(),
iter(per_test_output))
+ skipped_tests = list(filter(
+ lambda line: re.search('test_.* \(pyspark\..*\) ...
skipped ', line),
+ decoded_lines))
+ skipped_counts = len(skipped_tests)
+ if skipped_counts > 0:
+ key = (pyspark_python, test_name)
+ SKIPPED_TESTS[key] = skipped_tests
+ per_test_output.close()
+ except:
+ import traceback
+ print_red("\nGot an exception while trying to store "
+ "skipped test output:\n%s" % traceback.format_exc())
+ # Here, we use os._exit() instead of sys.exit() in order to
force Python to exit even if
+ # this code is invoked from a thread other than the main
thread.
+ os._exit(-1)
+ if skipped_counts != 0:
+ LOGGER.info(
+ "Finished test(%s): %s (%is) ... %s tests were skipped",
pyspark_python, test_name,
--- End diff --
Not sure if there's a better format. let me know.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
