Repository: incubator-livy Updated Branches: refs/heads/master 0bcc290dd -> 4cfb6bcb8
[LIVY-518][BUILD] Support Spark 2.4 ## What changes were proposed in this pull request? This PR proposes to support Spark 2.4 in Livy. ## How was this patch tested? Unit tests and manual tests. Author: hyukjinkwon <gurwls...@apache.org> Author: Hyukjin Kwon <gurwls...@apache.org> Closes #121 from HyukjinKwon/spark24-support. Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/4cfb6bcb Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/4cfb6bcb Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/4cfb6bcb Branch: refs/heads/master Commit: 4cfb6bcb8fb9ac6b2d6c8b3d04b20f647b507e1f Parents: 0bcc290 Author: hyukjinkwon <gurwls...@apache.org> Authored: Wed Nov 7 09:16:37 2018 -0800 Committer: Marcelo Vanzin <van...@cloudera.com> Committed: Wed Nov 7 09:16:37 2018 -0800 ---------------------------------------------------------------------- .travis.yml | 4 +++ README.md | 2 +- pom.xml | 32 ++++++++++++++++++++ .../org/apache/livy/rsc/TestSparkClient.java | 16 +++++++--- .../org/apache/livy/utils/LivySparkUtils.scala | 4 ++- 5 files changed, 52 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/4cfb6bcb/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index a72a875..d05e6dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,10 @@ matrix: env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipITs' - name: "Spark 2.3 ITs" env: MVN_FLAG='-Pspark-2.3 -Pthriftserver -DskipTests' + - name: "Spark 2.4 Unit Tests" + env: MVN_FLAG='-Pspark-2.4 -Pthriftserver -DskipITs' + - name: "Spark 2.4 ITs" + env: MVN_FLAG='-Pspark-2.4 -Pthriftserver -DskipTests' jdk: - oraclejdk8 http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/4cfb6bcb/README.md ---------------------------------------------------------------------- diff --git a/README.md b/README.md index 735afb3..a1f0854 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Required python packages for building Livy: To run Livy, you will also need a Spark installation. You can get Spark releases at https://spark.apache.org/downloads.html. -Livy requires Spark 2.2 or 2.3. You can switch to a different version of Spark by setting the +Livy requires Spark 2.2+. You can switch to a different version of Spark by setting the ``SPARK_HOME`` environment variable in the Livy server process, without needing to rebuild Livy. http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/4cfb6bcb/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 16b1bc0..e24827c 100644 --- a/pom.xml +++ b/pom.xml @@ -1042,6 +1042,38 @@ </profile> <profile> + <id>spark-2.4</id> + <activation> + <property> + <name>spark-2.4</name> + </property> + </activation> + <properties> + <spark.scala-2.11.version>2.4.0</spark.scala-2.11.version> + <spark.version>${spark.scala-2.11.version}</spark.version> + <netty.spark-2.11.version>4.1.17.Final</netty.spark-2.11.version> + <java.version>1.8</java.version> + <py4j.version>0.10.7</py4j.version> + <json4s.version>3.2.11</json4s.version> + </properties> + </profile> + + <profile> + <id>spark-2.4-it</id> + <activation> + <property> + <name>spark-2.4-it</name> + </property> + </activation> + <properties> + <spark.bin.download.url> + http://mirrors.advancedhosters.com/apache/spark/spark-2.4.0/spark-2.4.0-bin-hadoop2.7.tgz + </spark.bin.download.url> + <spark.bin.name>spark-2.4.0-bin-hadoop2.7</spark.bin.name> + </properties> + </profile> + + <profile> <id>skip-parent-modules</id> <activation> <file> http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/4cfb6bcb/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java ---------------------------------------------------------------------- diff --git a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java index aa4d319..bf0592b 100644 --- a/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java +++ b/rsc/src/test/java/org/apache/livy/rsc/TestSparkClient.java @@ -57,6 +57,10 @@ public class TestSparkClient { private static final long TIMEOUT = 100; private Properties createConf(boolean local) { + return createConf(local, true); + } + + private Properties createConf(boolean local, boolean hiveSupport) { Properties conf = new Properties(); if (local) { conf.put(CLIENT_IN_PROCESS.key(), "true"); @@ -72,8 +76,8 @@ public class TestSparkClient { conf.put(CLIENT_SHUTDOWN_TIMEOUT.key(), "30s"); conf.put(LIVY_JARS.key(), ""); - conf.put("spark.repl.enableHiveContext", "true"); - conf.put("spark.sql.catalogImplementation", "hive"); + conf.put("spark.repl.enableHiveContext", hiveSupport); + conf.put("spark.sql.catalogImplementation", hiveSupport ? "hive" : "in-memory"); conf.put(RETAINED_SHARE_VARIABLES.key(), "2"); return conf; } @@ -271,7 +275,7 @@ public class TestSparkClient { @Test public void testSparkSQLJob() throws Exception { - runTest(true, new TestFunction() { + runTest(true, false, new TestFunction() { @Override void call(LivyClient client) throws Exception { JobHandle<List<String>> handle = client.submit(new SQLGetTweets(false)); @@ -518,7 +522,11 @@ public class TestSparkClient { } private void runTest(boolean local, TestFunction test) throws Exception { - Properties conf = createConf(local); + runTest(local, true, test); + } + + private void runTest(boolean local, boolean hiveSupport, TestFunction test) throws Exception { + Properties conf = createConf(local, hiveSupport); LivyClient client = null; try { test.config(conf); http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/4cfb6bcb/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala ---------------------------------------------------------------------- diff --git a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala index 6097d32..dc98b0d 100644 --- a/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala +++ b/server/src/main/scala/org/apache/livy/utils/LivySparkUtils.scala @@ -30,6 +30,8 @@ object LivySparkUtils extends Logging { // For each Spark version we supported, we need to add this mapping relation in case Scala // version cannot be detected from "spark-submit --version". private val _defaultSparkScalaVersion = SortedMap( + // Spark 2.4 + Scala 2.11 + (2, 4) -> "2.11", // Spark 2.3 + Scala 2.11 (2, 3) -> "2.11", // Spark 2.2 + Scala 2.11 @@ -38,7 +40,7 @@ object LivySparkUtils extends Logging { // Supported Spark version private val MIN_VERSION = (2, 2) - private val MAX_VERSION = (2, 4) + private val MAX_VERSION = (3, 0) private val sparkVersionRegex = """version (.*)""".r.unanchored private val scalaVersionRegex = """Scala version (.*), Java""".r.unanchored