grundprinzip commented on code in PR #40993:
URL: https://github.com/apache/spark/pull/40993#discussion_r1182199493
##########
python/pyspark/sql/connect/session.py:
##########
@@ -123,14 +123,26 @@ def appName(self, name: str) -> "SparkSession.Builder":
def remote(self, location: str = "sc://localhost") ->
"SparkSession.Builder":
return self.config("spark.remote", location)
+ def channelBuilder(self, channelBuilder: ChannelBuilder) ->
"SparkSession.Builder":
+ with self._lock:
+ self._options["channelBuilder"] = channelBuilder
+ return self
+
def enableHiveSupport(self) -> "SparkSession.Builder":
raise NotImplementedError("enableHiveSupport not implemented for
Spark Connect")
def getOrCreate(self) -> "SparkSession":
global _active_spark_session
if _active_spark_session is not None:
return _active_spark_session
- _active_spark_session =
SparkSession(connectionString=self._options["spark.remote"])
+ if "channelBuilder" in self._options and "spark.remote" in
self._options:
+ raise ValueError("either connection string or ChannelBuilder
can be specified")
+ connection = "sc://localhost"
+ if "spark.remote" in self._options:
+ connection = self._options["spark.remote"]
+ elif "channelBuilder" in self._options:
+ connection = self._options["channelBuilder"]
Review Comment:
If neither is specified, this will set localhost as default which was not
the default before.
```suggestion
if "channelBuilder" in self._options:
_active_spark_session =
SparkSession(connection=self._options["channelBuilder"])
else:
_active_spark_session =
SparkSession(connection=self._options["spark.remote"])
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
