This is an automated email from the ASF dual-hosted git repository.
nicholasjiang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 9255e4ff8 [CELEBORN-1747] Fix flaky test - HybridShuffleWordCountTest
9255e4ff8 is described below
commit 9255e4ff874e7e6f8ddf4337bfb0c518875ea8bd
Author: Wang, Fei <[email protected]>
AuthorDate: Tue Nov 26 14:32:11 2024 +0800
[CELEBORN-1747] Fix flaky test - HybridShuffleWordCountTest
### What changes were proposed in this pull request?
Use `setupMiniClusterWithRandomPorts` instead of `setUpMiniCluster`, and
set `setUpMiniCluster` to be private.
### Why are the changes needed?
Fix flaky test.
```
HybridShuffleWordCountTest:
Nov 26, 2024 4:44:10 AM org.glassfish.jersey.server.wadl.WadlFeature
configure
WARNING: JAXBContext implementation could not be found. WADL feature is
disabled.
*** RUN ABORTED ***
java.io.IOException: Failed to bind to localhost/127.0.0.1:32886
at
org.eclipse.jetty.server.ServerConnector.openAcceptChannel(ServerConnector.java:349)
at org.eclipse.jetty.server.ServerConnector.open(ServerConnector.java:310)
at
org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:80)
at
org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:234)
at
org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
at
org.apache.celeborn.server.common.http.HttpServer.start(HttpServer.scala:45)
at
org.apache.celeborn.server.common.HttpService.startInternal(HttpService.scala:354)
at
org.apache.celeborn.server.common.HttpService.startHttpServer(HttpService.scala:210)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.createMaster(MiniClusterFeature.scala:115)
at
org.apache.celeborn.service.deploy.MiniClusterFeature.setUpMaster(MiniClusterFeature.scala:155)
...
Cause: java.net.BindException: Address already in use
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
GA.
Closes #2950 from turboFei/fix_flaky_test.
Authored-by: Wang, Fei <[email protected]>
Signed-off-by: SteNicholas <[email protected]>
---
.../org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala | 2 +-
.../scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala
b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala
index b174914ef..917132386 100644
---
a/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala
+++
b/tests/flink-it/src/test/scala/org/apache/celeborn/tests/flink/HybridShuffleWordCountTest.scala
@@ -50,7 +50,7 @@ class HybridShuffleWordCountTest extends AnyFunSuite with
Logging with MiniClust
"celeborn.master.host" -> "localhost",
"celeborn.master.port" -> "9097")
val workerConf = Map("celeborn.master.endpoints" -> "localhost:9097")
- workers = setUpMiniCluster(masterConf, workerConf)._2
+ workers = setupMiniClusterWithRandomPorts(masterConf, workerConf)._2
}
override def afterAll(): Unit = {
diff --git
a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
index bf5994527..3fd407284 100644
---
a/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
+++
b/worker/src/test/scala/org/apache/celeborn/service/deploy/MiniClusterFeature.scala
@@ -247,7 +247,7 @@ trait MiniClusterFeature extends Logging {
workerInfos.keySet
}
- def setUpMiniCluster(
+ private def setUpMiniCluster(
masterConf: Map[String, String] = null,
workerConf: Map[String, String] = null,
workerNum: Int = 3): (Master, collection.Set[Worker]) = {
