HyukjinKwon commented on code in PR #44504:
URL: https://github.com/apache/spark/pull/44504#discussion_r1437292785
##########
core/src/main/scala/org/apache/spark/api/python/PythonUtils.scala:
##########
@@ -145,4 +149,48 @@ private[spark] object PythonUtils extends Logging {
listOfPackages.foreach(x => logInfo(s"List of Python packages :-
${formatOutput(x)}"))
}
}
+
+ // Only for testing.
+ private[spark] var additionalTestingPath: Option[String] = None
+
+ private[spark] def createPythonFunction(command: Array[Byte]):
SimplePythonFunction = {
+ val pythonExec: String = sys.env.getOrElse(
+ "PYSPARK_DRIVER_PYTHON", sys.env.getOrElse("PYSPARK_PYTHON", "python3"))
+
+ val sourcePython = if (Utils.isTesting) {
+ // Put PySpark source code instead of the build zip archive so we don't
need
+ // to build PySpark every time during development.
+ val sparkHome: String = {
+ require(
+ sys.props.contains("spark.test.home") ||
sys.env.contains("SPARK_HOME"),
+ "spark.test.home or SPARK_HOME is not set.")
+ sys.props.getOrElse("spark.test.home", sys.env("SPARK_HOME"))
+ }
+ val sourcePath = Paths.get(sparkHome, "python").toAbsolutePath
+ val py4jPath = Paths.get(
+ sparkHome, "python", "lib", PythonUtils.PY4J_ZIP_NAME).toAbsolutePath
Review Comment:
We don't use it but we need it when we import .. e.g.,
https://github.com/apache/spark/blob/master/python/pyspark/__init__.py#L53 ->
https://github.com/apache/spark/blob/master/python/pyspark/conf.py#L23
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
