Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20473#discussion_r165562831
--- Diff: python/run-tests.py ---
@@ -151,6 +152,61 @@ def parse_opts():
return opts
+def _check_dependencies(python_exec, modules_to_test):
+ if "COVERAGE_PROCESS_START" in os.environ:
+ # Make sure if coverage is installed.
+ try:
+ subprocess_check_output(
+ [python_exec, "-c", "import coverage"],
+ stderr=open(os.devnull, 'w'))
+ except:
+ print_red("Coverage is not installed in Python executable '%s'
"
+ "but 'COVERAGE_PROCESS_START' environment variable
is set, "
+ "exiting." % python_exec)
+ sys.exit(-1)
+
+ if pyspark_sql in modules_to_test:
+ # If we should test 'pyspark-sql', it checks if PyArrow and Pandas
are installed and
+ # explicitly prints out. See SPARK-23300.
+ try:
+ pyarrow_version = subprocess_check_output(
+ [python_exec, "-c", "import pyarrow;
print(pyarrow.__version__)"],
+ universal_newlines=True,
+ stderr=open(os.devnull, 'w')).strip()
+ if LooseVersion(pyarrow_version) >= LooseVersion('0.8.0'):
+ LOGGER.info("Will test PyArrow related features against
Python executable "
+ "'%s' in '%s' module." % (python_exec,
pyspark_sql.name))
+ else:
+ LOGGER.warning(
+ "Will skip PyArrow related features against Python
executable "
+ "'%s' in '%s' module. PyArrow >= 0.8.0 is required;
however, PyArrow "
+ "%s was found." % (python_exec, pyspark_sql.name,
pyarrow_version))
+ except:
+ LOGGER.warning(
+ "Will skip PyArrow related features against Python
executable "
+ "'%s' in '%s' module. PyArrow >= 0.8.0 is required;
however, PyArrow "
+ "was not found." % (python_exec, pyspark_sql.name))
+
+ try:
+ pandas_version = subprocess_check_output(
+ [python_exec, "-c", "import pandas;
print(pandas.__version__)"],
+ universal_newlines=True,
+ stderr=open(os.devnull, 'w')).strip()
+ if LooseVersion(pandas_version) >= LooseVersion('0.19.2'):
--- End diff --
ditto.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
