ueshin commented on a change in pull request #29703:
URL: https://github.com/apache/spark/pull/29703#discussion_r492417508
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
We should insert this entry between `"../"` and
`os.path.dirname(os.path.realpath(__file__))`, which is the module home;
otherwise, `os.path.dirname(os.path.realpath(__file__))` will be the spark home.
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
We should insert this entry between `"../"` and
`os.path.dirname(os.path.realpath(__file__))`, which is the same as the module
home if pip is used for the installation; otherwise,
`os.path.dirname(os.path.realpath(__file__))` will be the spark home.
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
We should insert this entry between `"../"` and
`os.path.dirname(os.path.realpath(__file__))`, which is the same as the module
home if pip is used for the installation; otherwise, the module home will be
the spark home and the distribution under `spark-distribution` is not used.
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
We should insert this entry at first or between `"../"` and
`os.path.dirname(os.path.realpath(__file__))`, which is the same as the module
home if pip is used for the installation; otherwise, the module home will be
the spark home and the distribution under `spark-distribution` is not used.
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
We should insert this entry before
`os.path.dirname(os.path.realpath(__file__))`, which is the same as the module
home if pip is used for the installation; otherwise, the module home will be
the spark home and the distribution under `spark-distribution` is not used.
##########
File path: python/pyspark/find_spark_home.py
##########
@@ -42,7 +42,11 @@ def is_spark_home(path):
import_error_raised = False
from importlib.util import find_spec
try:
+ # Spark distribution can be downloaded when HADOOP_VERSION environment
variable is set.
+ # We should look up this directory first, see also SPARK-32017.
+ spark_dist_dir = "spark-distribution"
module_home = os.path.dirname(find_spec("pyspark").origin)
+ paths.append(os.path.join(module_home, spark_dist_dir))
Review comment:
The function in the file `find_spark_home.py` is called from
`launch_gateway`, so if the users initialize Spark by themselves in their
Python repr, the path will be the module home?
```py
$ python
>>> from pyspark.sql import SparkSession
>>> spark = SparkSession.builder.getOrCreate()
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
