This is an automated email from the ASF dual-hosted git repository.
Yicong-Huang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 29d94a9951c8 [SPARK-60230][ML][PYTHON] Keep TorchDistributor
MASTER_PORT reserved until training finishes
29d94a9951c8 is described below
commit 29d94a9951c82243e1534f63b2de444c996ee5c8
Author: Yicong Huang <[email protected]>
AuthorDate: Sat Oct 24 19:32:10 2026 +0000
[SPARK-60230][ML][PYTHON] Keep TorchDistributor MASTER_PORT reserved until
training finishes
### What changes were proposed in this pull request?
In distributed mode, `TorchDistributor` picks `MASTER_PORT` in partition 0
by binding a socket to port 0 and releasing it right away. This PR adds
`TorchDistributor._reserve_free_port`, which binds the socket with
`SO_REUSEADDR` (without listening), and keeps it open until training finishes.
The socket is closed in the existing `finally` block of the barrier task.
While the socket is held, binds without `SO_REUSEADDR` on the same address
or the wildcard address fail, and the kernel does not hand the port out for
outgoing connections. A socket that sets `SO_REUSEADDR`, or one bound to
another local address (e.g. loopback), can still take it. torchrun's `TCPStore`
also sets `SO_REUSEADDR`, so it can still listen on the port.
### Why are the changes needed?
`pyspark.ml.tests.connect.test_connect_pipeline`
`PipelineTestsOnConnect.test_pipeline` is flaky in CI:
```
torch.distributed.DistNetworkError: The server socket has failed to listen
on any local network address. port: 39055, useIpv6: false, code: -98, name:
EADDRINUSE, message: address already in use
```
The port comes from the ephemeral range, and between releasing it and
torchrun binding it, another socket (e.g. an outgoing connection on the shared
CI runner) can take it.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added `test_reserve_free_port`, which checks that a competing bind fails
while the port is reserved and that a `TCPStore` can still listen on it. Added
`test_master_port_reserved_until_training_finishes`, which checks the port
stays reserved during training and is released afterwards, on both success and
failure. Ran `pyspark.ml.torch.tests.test_distributor`,
`pyspark.ml.tests.connect.test_parity_torch_distributor`,
`pyspark.ml.tests.connect.test_legacy_mode_pipeline` and `pyspark.ml. [...]
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
This pull request and its description were written by Isaac.
Closes #59451 from Yicong-Huang/SPARK-60230.
Authored-by: Yicong Huang <[email protected]>
Signed-off-by: Yicong-Huang <[email protected]>
---
python/pyspark/ml/torch/distributor.py | 32 +++++++--
python/pyspark/ml/torch/tests/test_distributor.py | 80 +++++++++++++++++++++++
2 files changed, 108 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/ml/torch/distributor.py
b/python/pyspark/ml/torch/distributor.py
index 63912e80e6c4..a6c34b9643cd 100644
--- a/python/pyspark/ml/torch/distributor.py
+++ b/python/pyspark/ml/torch/distributor.py
@@ -21,6 +21,7 @@ import os
import random
import re
import shutil
+import socket
import subprocess
import sys
import tempfile
@@ -448,6 +449,26 @@ class TorchDistributor(Distributor):
]
return torchrun_args, processes_per_node
+ @staticmethod
+ def _reserve_free_port(address: str) -> socket.socket:
+ """
+ Bind a socket to a free port on the given address and return it.
+
+ Closing the socket right away would let another socket take the port
before
+ torchrun's TCPStore binds it. The returned socket is bound but not
listening and
+ sets SO_REUSEADDR, so TCPStore, which also sets SO_REUSEADDR, can
still listen on
+ the port while the socket keeps it reserved.
+
+ The reservation only blocks binds that do not set SO_REUSEADDR on the
same address
+ or the wildcard address, and outgoing connections picking the port. A
socket that
+ sets SO_REUSEADDR, or one bound to another local address such as
loopback, can
+ still take the port.
+ """
+ sock = socket.socket()
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ sock.bind((address, 0))
+ return sock
+
@staticmethod
def _create_torchrun_command(
input_params: Dict[str, Any], path_to_train_file: str, *args: Any
@@ -685,14 +706,15 @@ class TorchDistributor(Distributor):
CUDA_VISIBLE_DEVICES = "CUDA_VISIBLE_DEVICES"
+ # Sockets reserving MASTER_PORT, held until training finishes.
+ reserved_port_sockets = []
+
def get_free_port(address: str, context: "BarrierTaskContext") ->
int:
port = ""
if context.partitionId() == 0:
try:
- import socket
-
- sock = socket.socket()
- sock.bind((address, 0))
+ sock = TorchDistributor._reserve_free_port(address)
+ reserved_port_sockets.append(sock)
port = sock.getsockname()[1]
except socket.error:
pass
@@ -766,6 +788,8 @@ class TorchDistributor(Distributor):
**kwargs,
)
finally:
+ for sock in reserved_port_sockets:
+ sock.close()
try:
LogStreamingClient._destroy()
except BaseException:
diff --git a/python/pyspark/ml/torch/tests/test_distributor.py
b/python/pyspark/ml/torch/tests/test_distributor.py
index af994c81af40..3ff3ba1810f7 100644
--- a/python/pyspark/ml/torch/tests/test_distributor.py
+++ b/python/pyspark/ml/torch/tests/test_distributor.py
@@ -300,6 +300,86 @@ class TorchDistributorBaselineUnitTestsMixin:
self.assertIn("Start torch distributor log streaming server failed",
message)
self.assertIn("Address already in use", message)
+ def test_reserve_free_port(self) -> None:
+ import socket
+ from datetime import timedelta
+
+ from torch.distributed import TCPStore
+
+ sock = TorchDistributor._reserve_free_port("127.0.0.1")
+ try:
+ port = sock.getsockname()[1]
+ # Plain binds on the same address cannot take the reserved port.
+ other = socket.socket()
+ try:
+ with self.assertRaises(OSError):
+ other.bind(("127.0.0.1", port))
+ finally:
+ other.close()
+ # torchrun's TCPStore can still listen on it.
+ store = TCPStore("127.0.0.1", port, 1, True,
timeout=timedelta(seconds=30))
+ store.set("key", "value")
+ self.assertEqual(store.get("key"), b"value")
+ finally:
+ sock.close()
+
+ def test_master_port_reserved_until_training_finishes(self) -> None:
+ import socket
+
+ from pyspark.taskcontext import BarrierTaskContext
+
+ def bind_master_port() -> None:
+ sock = socket.socket()
+ try:
+ sock.bind((os.environ["MASTER_ADDR"],
int(os.environ["MASTER_PORT"])))
+ finally:
+ sock.close()
+
+ def train_and_check_reserved(*args: Any, **kwargs: Any) -> str:
+ with self.assertRaises(OSError):
+ bind_master_port()
+ return "done"
+
+ def train_check_reserved_and_raise(*args: Any, **kwargs: Any) -> str:
+ train_and_check_reserved()
+ raise RuntimeError("training failed")
+
+ dist = TorchDistributor(num_processes=1, local_mode=False,
use_gpu=False)
+ dist.driver_address = "127.0.0.1"
+ dist.log_streaming_server_port = -1
+ dist.log_streaming_auth_secret = None
+ context = MagicMock()
+ context.partitionId.return_value = 0
+ context.getTaskInfos.return_value = [MagicMock(address="127.0.0.1:0")]
+ context.allGather.side_effect = lambda message: [message]
+
+ for train_fn in [train_and_check_reserved,
train_check_reserved_and_raise]:
+ with (
+ self.subTest(train_fn=train_fn.__name__),
+ patch.dict(os.environ),
+ patch.object(BarrierTaskContext, "get", return_value=context),
+ patch("pyspark.ml.torch.distributor.LogStreamingClient"),
+ patch.object(
+ TorchDistributor, "_get_output_from_framework_wrapper",
side_effect=train_fn
+ ),
+ ):
+ task_fn = dist._get_spark_task_function(None, MagicMock(),
None, None)
+ gen = task_fn(iter([]))
+ # The reservation is released explicitly, while the generator
or the
+ # exception still keeps the reserving socket alive.
+ if train_fn is train_check_reserved_and_raise:
+ with self.assertRaisesRegex(RuntimeError, "training
failed"):
+ try:
+ next(gen)
+ except RuntimeError:
+ bind_master_port()
+ raise
+ else:
+ # Partition 0's output batch, yielded after the finally.
+ next(gen)
+ bind_master_port()
+ self.assertEqual(list(gen), [])
+
def test_create_torchrun_command(self) -> None:
train_path = "train.py"
args_string = ["1", "3"]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]