This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.4 by this push:
new d0eaebbd7f9 [SPARK-45508][CORE] Add
"--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" so Platform can access
Cleaner on Java 9+
d0eaebbd7f9 is described below
commit d0eaebbd7f90fd121c7f97ca376b7141ad15731b
Author: Josh Rosen <joshro...@databricks.com>
AuthorDate: Fri Oct 13 13:29:45 2023 +0800
[SPARK-45508][CORE] Add
"--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED" so Platform can access
Cleaner on Java 9+
This PR adds `--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED` to our
JVM flags so that we can access `jdk.internal.ref.Cleaner` in JDK 9+.
This allows Spark to allocate direct memory while ignoring the JVM's
MaxDirectMemorySize limit. Spark uses JDK internal APIs to directly construct
DirectByteBuffers while bypassing that limit, but there is a fallback path at
https://github.com/apache/spark/blob/v3.5.0/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java#L213
that is used if we cannot reflectively access the `Cleaner` API.
No.
Added a unit test in `PlatformUtilSuite`.
No.
Closes #43344 from JoshRosen/SPARK-45508.
Authored-by: Josh Rosen <joshro...@databricks.com>
Signed-off-by: yangjie01 <yangji...@baidu.com>
(cherry picked from commit 96bac6c033b5bb37101ebcd8436ab9a84db8e092)
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
(cherry picked from commit 146fba1a22e3f1555f3e4494522810030f9a7854)
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java | 7 ++++++-
.../src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java | 7 +++++++
.../src/main/java/org/apache/spark/launcher/JavaModuleOptions.java | 1 +
pom.xml | 1 +
project/SparkBuild.scala | 1 +
5 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
index 12867627379..51fdeb9f9ed 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java
@@ -94,7 +94,7 @@ public final class Platform {
Method createMethod = cleanerClass.getMethod("create", Object.class,
Runnable.class);
// Accessing jdk.internal.ref.Cleaner should actually fail by default
in JDK 9+,
// unfortunately, unless the user has allowed access with something
like
- // --add-opens java.base/java.lang=ALL-UNNAMED If not, we can't
really use the Cleaner
+ // --add-opens java.base/jdk.internal.ref=ALL-UNNAMED If not, we
can't use the Cleaner
// hack below. It doesn't break, just means the user might run into
the default JVM limit
// on off-heap memory and increase it or set the flag above. This
tests whether it's
// available:
@@ -116,6 +116,11 @@ public final class Platform {
}
}
+ // Visible for testing
+ public static boolean cleanerCreateMethodIsDefined() {
+ return CLEANER_CREATE_METHOD != null;
+ }
+
/**
* @return true when running JVM is having sun's Unsafe package available in
it and underlying
* system having unaligned-access capability.
diff --git
a/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java
b/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java
index c59878fea99..c99f2d85f4e 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/PlatformUtilSuite.java
@@ -157,4 +157,11 @@ public class PlatformUtilSuite {
Assert.assertEquals(1024 * 1024 + 7, onheap4.size());
Assert.assertEquals(obj3, onheap4.getBaseObject());
}
+
+ @Test
+ public void cleanerCreateMethodIsDefined() {
+ // Regression test for SPARK-45508: we don't expect the "no cleaner"
fallback
+ // path to be hit in normal usage.
+ Assert.assertTrue(Platform.cleanerCreateMethodIsDefined());
+ }
}
diff --git
a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
index 013dde2766f..f6a9607e7c5 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
@@ -36,6 +36,7 @@ public class JavaModuleOptions {
"--add-opens=java.base/java.util=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+ "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
"--add-opens=java.base/sun.security.action=ALL-UNNAMED",
diff --git a/pom.xml b/pom.xml
index 12fe9b50248..6fa84df3d25 100644
--- a/pom.xml
+++ b/pom.xml
@@ -317,6 +317,7 @@
--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
--add-opens=java.base/sun.security.action=ALL-UNNAMED
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 05b88bc8b14..c71df993ae2 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -1591,6 +1591,7 @@ object TestSettings {
"--add-opens=java.base/java.util=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent=ALL-UNNAMED",
"--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED",
+ "--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.ch=ALL-UNNAMED",
"--add-opens=java.base/sun.nio.cs=ALL-UNNAMED",
"--add-opens=java.base/sun.security.action=ALL-UNNAMED",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
