This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3e1c3bdc7ee1 [SPARK-54009][CORE] Support `spark.io.mode.default`
3e1c3bdc7ee1 is described below
commit 3e1c3bdc7ee19d839fde177d95222030df1b27f9
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Thu Oct 23 18:54:54 2025 -0700
[SPARK-54009][CORE] Support `spark.io.mode.default`
### What changes were proposed in this pull request?
This PR aims to support a new configuration `spark.io.mode.default` to
control the default value for all modules' configurations `spark.*.io.mode`.
For example, `spark.rpc.io.mode` and `spark.shuffle.io.mode`.
### Why are the changes needed?
Currently, the default value of `spark.*.io.mode` is hard-coded to `NIO`
which means we need to change all configurations via `spark.*.io.mode` pattern.
https://github.com/apache/spark/blob/b46cf270abab0daaee4c504951fdf3cc2920bb76/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java#L91
This PR aims to help users to change all modules' default easily via a
single configuration.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with the newly added test case.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #52717 from dongjoon-hyun/SPARK-54009.
Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../org/apache/spark/network/util/NettyUtils.java | 3 ++-
.../apache/spark/network/util/TransportConf.java | 6 ++++--
.../apache/spark/network/TransportConfSuite.java | 22 ++++++++++++++++++++++
3 files changed, 28 insertions(+), 3 deletions(-)
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
index da4b3109bbe1..1c9c15552f8b 100644
---
a/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/NettyUtils.java
@@ -36,7 +36,8 @@ import io.netty.util.concurrent.DefaultThreadFactory;
import io.netty.util.internal.PlatformDependent;
/**
- * Utilities for creating various Netty constructs based on whether we're
using EPOLL or NIO.
+ * Utilities for creating various Netty constructs based on whether we're
using NIO, EPOLL,
+ * or KQUEUE.
*/
public class NettyUtils {
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
index 003e72edf29e..5718c20c7d11 100644
---
a/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/TransportConf.java
@@ -28,6 +28,7 @@ import io.netty.util.NettyRuntime;
*/
public class TransportConf {
+ private final String SPARK_NETWORK_DEFAULT_IO_MODE_KEY =
"spark.io.mode.default";
private final String SPARK_NETWORK_IO_MODE_KEY;
private final String SPARK_NETWORK_IO_PREFERDIRECTBUFS_KEY;
private final String SPARK_NETWORK_IO_CONNECTIONTIMEOUT_KEY;
@@ -86,9 +87,10 @@ public class TransportConf {
return module;
}
- /** IO mode: nio or epoll */
+ /** IO mode: NIO, EPOLL, or KQUEUE */
public String ioMode() {
- return conf.get(SPARK_NETWORK_IO_MODE_KEY, "NIO").toUpperCase(Locale.ROOT);
+ String defaultIOMode = conf.get(SPARK_NETWORK_DEFAULT_IO_MODE_KEY, "NIO");
+ return conf.get(SPARK_NETWORK_IO_MODE_KEY,
defaultIOMode).toUpperCase(Locale.ROOT);
}
/** If true, we will prefer allocating off-heap byte buffers within Netty. */
diff --git
a/common/network-common/src/test/java/org/apache/spark/network/TransportConfSuite.java
b/common/network-common/src/test/java/org/apache/spark/network/TransportConfSuite.java
index 1537f67e98d1..f77a93e6247b 100644
---
a/common/network-common/src/test/java/org/apache/spark/network/TransportConfSuite.java
+++
b/common/network-common/src/test/java/org/apache/spark/network/TransportConfSuite.java
@@ -17,10 +17,12 @@
package org.apache.spark.network;
import java.io.File;
+import java.util.Map;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
+import org.apache.spark.network.util.MapConfigProvider;
import org.apache.spark.network.util.TransportConf;
import org.apache.spark.network.ssl.SslSampleConfigs;
@@ -85,4 +87,24 @@ public class TransportConfSuite {
public void testSsltrustStoreReloadIntervalMs() {
assertEquals(10000, transportConf.sslRpctrustStoreReloadIntervalMs());
}
+
+ @Test
+ public void testDefaultIOMode() {
+ TransportConf c1 = new TransportConf("m1", new
MapConfigProvider(Map.of()));
+ assertEquals("NIO", c1.ioMode());
+
+ TransportConf c2 = new TransportConf("m1",
+ new MapConfigProvider(Map.of("spark.io.mode.default", "KQUEUE")));
+ assertEquals("KQUEUE", c2.ioMode());
+
+ TransportConf c3 = new TransportConf("m2",
+ new MapConfigProvider(Map.of("spark.io.mode.default", "KQUEUE")));
+ assertEquals("KQUEUE", c3.ioMode());
+
+ TransportConf c4 = new TransportConf("m3",
+ new MapConfigProvider(Map.of(
+ "spark.io.mode.default", "KQUEUE",
+ "spark.m3.io.mode", "EPOLL")));
+ assertEquals("EPOLL", c4.ioMode());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
