This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new da2ae29c9cab [SPARK-47215][CORE][TESTS] Reduce the number of required
threads in `MasterSuite`
da2ae29c9cab is described below
commit da2ae29c9cabe336f95ab3737e97aa8a5bd33ada
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Wed Feb 28 16:21:26 2024 -0800
[SPARK-47215][CORE][TESTS] Reduce the number of required threads in
`MasterSuite`
### What changes were proposed in this pull request?
This PR aims to reduce the umber of required threads in `MasterSuite` test.
### Why are the changes needed?
-
https://github.com/apache/spark/actions/workflows/build_maven_java21_macos14.yml
- https://github.com/apache/spark/actions/runs/8070641575/job/22086547398
```
- SPARK-46881: scheduling with workerSelectionPolicy - CORES_FREE_ASC (true)
- SPARK-46881: scheduling with workerSelectionPolicy - CORES_FREE_ASC
(false)
Warning: [3943.730s][warning][os,thread] Failed to start thread "Unknown
thread" - pthread_create failed (EAGAIN) for attributes: stacksize: 4096k,
guardsize: 16k, detached.
Warning: [3943.730s][warning][os,thread] Failed to start the native thread
for java.lang.Thread "rpc-server-13566-3"
Warning: [3943.730s][warning][os,thread] Failed to start thread "Unknown
thread" - pthread_create failed (EAGAIN) for attributes: stacksize: 4096k,
guardsize: 16k, detached.
Warning: [3943.730s][warning][os,thread] Failed to start the native thread
for java.lang.Thread "globalEventExecutor-3-961"
*** RUN ABORTED ***
An exception or error caused a run to abort: unable to create native
thread: possibly out of memory or process/resource limits reached
java.lang.OutOfMemoryError: unable to create native thread: possibly out
of memory or process/resource limits reached
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #45320 from dongjoon-hyun/SPARK-47215.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../scala/org/apache/spark/deploy/master/MasterSuite.scala | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git
a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
index 9992c2020f27..b4981ca3d9c6 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala
@@ -687,9 +687,9 @@ class MasterSuite extends SparkFunSuite
private val workerSelectionPolicyTestCases = Seq(
(CORES_FREE_ASC, true, List("10001", "10002")),
(CORES_FREE_ASC, false, List("10001")),
- (CORES_FREE_DESC, true, List("10004", "10005")),
- (CORES_FREE_DESC, false, List("10005")),
- (MEMORY_FREE_ASC, true, List("10001", "10005")),
+ (CORES_FREE_DESC, true, List("10002", "10003")),
+ (CORES_FREE_DESC, false, List("10003")),
+ (MEMORY_FREE_ASC, true, List("10001", "10003")),
(MEMORY_FREE_ASC, false, List("10001")),
(MEMORY_FREE_DESC, true, List("10002", "10003")),
(MEMORY_FREE_DESC, false, Seq("10002")),
@@ -701,11 +701,14 @@ class MasterSuite extends SparkFunSuite
val conf = new SparkConf()
.set(WORKER_SELECTION_POLICY.key, policy.toString)
.set(SPREAD_OUT_APPS.key, spreadOut.toString)
+ .set(UI_ENABLED.key, "false")
+ .set(Network.RPC_NETTY_DISPATCHER_NUM_THREADS, 1)
+ .set(Network.RPC_IO_THREADS, 1)
val master = makeAliveMaster(conf)
// Use different core and memory values to simplify the tests
MockWorker.counter.set(10000)
- (1 to 5).foreach { idx =>
+ (1 to 3).foreach { idx =>
val worker = new MockWorker(master.self, conf)
worker.rpcEnv.setupEndpoint(s"worker-$idx", worker)
val workerReg = RegisterWorker(
@@ -713,7 +716,7 @@ class MasterSuite extends SparkFunSuite
"localhost",
worker.self.address.port,
worker.self,
- idx * 10,
+ 4 + idx,
10240 * (if (idx < 2) idx else (6 - idx)),
"http://localhost:8080";,
RpcAddress("localhost", 10000))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
