ericm-db commented on code in PR #56907:
URL: https://github.com/apache/spark/pull/56907#discussion_r3547846726
##########
python/pyspark/sql/connect/session.py:
##########
@@ -1247,6 +1249,372 @@ def _start_connect_server(master: str, opts: Dict[str,
Any]) -> None:
messageParameters={},
)
+ # Opt-in reuse of a persistent local Spark Connect server. By default
``.remote("local[*]")``
+ # boots a fresh in-process server every process (see
``_start_connect_server`` above); when
+ # reuse is enabled, the first run starts a detached server
(``connect/local_server.py``) and
+ # records it in a discovery file, and later runs reconnect to it instead
of re-paying the cold
+ # start.
+
+ @staticmethod
+ def _local_connect_discovery_path() -> str:
+ """Location of the discovery file describing the running persistent
local server."""
+ override = os.environ.get("SPARK_LOCAL_CONNECT_DISCOVERY")
+ if override:
+ return override
+ return os.path.join(os.path.expanduser("~"), ".spark",
"connect-local.json")
+
+ @staticmethod
+ def _read_local_connect_discovery() -> Optional[Dict[str, Any]]:
+ """Read and validate the discovery file, returning ``None`` if it is
absent or malformed."""
+ path = SparkSession._local_connect_discovery_path()
+ try:
+ with open(path, "r") as f:
+ disc = json.load(f)
+ except (OSError, ValueError):
+ return None
+ if not isinstance(disc, dict) or not all(
+ k in disc for k in ("host", "port", "token", "pid",
"spark_version")
+ ):
+ return None
+ return disc
+
+ @staticmethod
+ def _local_connect_server_is_reusable(disc: Dict[str, Any]) -> bool:
+ """Decide whether the server described by ``disc`` can be reused by
this process.
+
+ Reuse requires that the recorded Spark version matches this client's,
the recorded process
+ is still alive, and it is accepting connections on the recorded port.
A version mismatch,
+ dead pid, or closed port means we must start our own server instead.
+ """
+ import socket
+ from pyspark.version import __version__
+
+ if disc.get("spark_version") != __version__:
Review Comment:
Made consistent.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
