Repository: spark
Updated Branches:
refs/heads/master 323cd92b9 -> f3d65cd0b
[SPARK-3068]remove MaxPermSize option for jvm 1.8
In JVM 1.8.0, MaxPermSize is no longer supported.
In spark `stderr` output, there would be a line of
Java HotSpot(TM) 64-Bit Server VM warning: ignoring option
MaxPermSize=128m; support was removed in 8.0
Author: Daoyuan Wang <[email protected]>
Closes #2011 from adrian-wang/maxpermsize and squashes the following commits:
ef1d660 [Daoyuan Wang] direct get java version in runtime
37db9c1 [Daoyuan Wang] code refine
3c1d554 [Daoyuan Wang] remove MaxPermSize option for jvm 1.8
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f3d65cd0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f3d65cd0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f3d65cd0
Branch: refs/heads/master
Commit: f3d65cd0bf34d8c362da492472c61796ef059428
Parents: 323cd92
Author: Daoyuan Wang <[email protected]>
Authored: Sat Aug 23 08:09:30 2014 -0700
Committer: Patrick Wendell <[email protected]>
Committed: Sat Aug 23 08:09:30 2014 -0700
----------------------------------------------------------------------
bin/spark-class | 7 ++++++-
bin/spark-class2.cmd | 8 +++++++-
.../scala/org/apache/spark/deploy/worker/CommandUtils.scala | 4 ++--
3 files changed, 15 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f3d65cd0/bin/spark-class
----------------------------------------------------------------------
diff --git a/bin/spark-class b/bin/spark-class
index 22acf92..c654354 100755
--- a/bin/spark-class
+++ b/bin/spark-class
@@ -105,9 +105,14 @@ else
exit 1
fi
fi
+JAVA_VERSION=$($RUNNER -version 2>&1 | sed 's/java version
"\(.*\)\.\(.*\)\..*"/\1\2/; 1q')
# Set JAVA_OPTS to be able to load native libraries and to set heap size
-JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
+if [ "$JAVA_VERSION" -ge 18 ]; then
+ JAVA_OPTS="$OUR_JAVA_OPTS"
+else
+ JAVA_OPTS="-XX:MaxPermSize=128m $OUR_JAVA_OPTS"
+fi
JAVA_OPTS="$JAVA_OPTS -Xms$OUR_JAVA_MEM -Xmx$OUR_JAVA_MEM"
# Load extra JAVA_OPTS from conf/java-opts, if it exists
http://git-wip-us.apache.org/repos/asf/spark/blob/f3d65cd0/bin/spark-class2.cmd
----------------------------------------------------------------------
diff --git a/bin/spark-class2.cmd b/bin/spark-class2.cmd
index e420eb4..e2c5f9c 100755
--- a/bin/spark-class2.cmd
+++ b/bin/spark-class2.cmd
@@ -77,7 +77,13 @@ rem All drivers use SPARK_JAVA_OPTS + SPARK_DRIVER_MEMORY.
The repl also uses SP
)
rem Set JAVA_OPTS to be able to load native libraries and to set heap size
-set JAVA_OPTS=-XX:MaxPermSize=128m %OUR_JAVA_OPTS%
-Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+for /f "tokens=3" %%i in ('java -version 2^>^&1 ^| find "version"') do set
jversion=%%i
+for /f "tokens=1 delims=_" %%i in ("%jversion:~1,-1%") do set jversion=%%i
+if "%jversion%" geq "1.8.0" (
+ set JAVA_OPTS=%OUR_JAVA_OPTS% -Djava.library.path=%SPARK_LIBRARY_PATH%
-Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+) else (
+ set JAVA_OPTS=-XX:MaxPermSize=128m %OUR_JAVA_OPTS%
-Djava.library.path=%SPARK_LIBRARY_PATH% -Xms%OUR_JAVA_MEM% -Xmx%OUR_JAVA_MEM%
+)
rem Attention: when changing the way the JAVA_OPTS are assembled, the change
must be reflected in CommandUtils.scala!
rem Test whether the user has built Spark
http://git-wip-us.apache.org/repos/asf/spark/blob/f3d65cd0/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
----------------------------------------------------------------------
diff --git
a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
index 687e492..12e98fd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/CommandUtils.scala
@@ -64,8 +64,6 @@ object CommandUtils extends Logging {
Seq()
}
- val permGenOpt = Seq("-XX:MaxPermSize=128m")
-
// Figure out our classpath with the external compute-classpath script
val ext = if (System.getProperty("os.name").startsWith("Windows")) ".cmd"
else ".sh"
val classPath = Utils.executeAndGetOutput(
@@ -73,6 +71,8 @@ object CommandUtils extends Logging {
extraEnvironment = command.environment)
val userClassPath = command.classPathEntries ++ Seq(classPath)
+ val javaVersion = System.getProperty("java.version")
+ val permGenOpt = if (!javaVersion.startsWith("1.8"))
Some("-XX:MaxPermSize=128m") else None
Seq("-cp",
userClassPath.filterNot(_.isEmpty).mkString(File.pathSeparator)) ++
permGenOpt ++ libraryOpts ++ workerLocalOpts ++ command.javaOpts ++
memoryOpts
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
