Repository: zeppelin Updated Branches: refs/heads/branch-0.6 5a270d43b -> 81ce3e641
[ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5 ### What is this PR for? Spark2 version of HDP 2.5 is "2.0.0.2.5.0.0-1245". Currently, Zeppelin parses this version to integer 2002500, which is higher than version "2.0.0" (i.e. 200 in integer). Therefore, Zeppelin thinks it's not supported version and fail. I fixed some codes to consider major, minor, and patch version. Now, version parsed using only first 3 numbers (major, minor, patch) and parsed into a 5-digit integer. For example, 2.0.0 -> 20000, 1.6.2 -> 10602, 1.10.1 -> 11001. This makes HDP 2.5 compatible, and also make it more robust than before. Since maybe Spark's minor version can be a 2-digit number in the future ### What type of PR is it? [Bug Fix] ### What is the Jira issue? [ZEPPELIN-1390](https://issues.apache.org/jira/browse/ZEPPELIN-1390) ### How should this be tested? I added SparkVersion test codes. Author: Jun <[email protected]> Closes #1381 from tae-jun/ZEPPELIN-1390 and squashes the following commits: bae5264 [Jun] Merge remote-tracking branch 'origin/master' into ZEPPELIN-1390 f5531ab [Jun] [ZEPPELIN-1390] Fix Python and R number versions 6ad8049 [Jun] [ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5 (cherry picked from commit d005c7967a57a572cc6d6620185ac6aedbfc56e3) Signed-off-by: Mina Lee <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/81ce3e64 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/81ce3e64 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/81ce3e64 Branch: refs/heads/branch-0.6 Commit: 81ce3e641d223252495eeeb527d2aab4332366ed Parents: 5a270d4 Author: Jun <[email protected]> Authored: Sun Sep 4 10:53:32 2016 +0900 Committer: Mina Lee <[email protected]> Committed: Fri Sep 9 13:08:33 2016 +0200 ---------------------------------------------------------------------- .../main/java/org/apache/zeppelin/spark/SparkVersion.java | 10 ++++++++-- spark/src/main/resources/R/zeppelin_sparkr.R | 2 +- spark/src/main/resources/python/zeppelin_pyspark.py | 6 +++--- .../java/org/apache/zeppelin/spark/SparkVersionTest.java | 9 +++++++-- 4 files changed, 19 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java index 17f2de7..c8a7c79 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java @@ -52,13 +52,19 @@ public class SparkVersion { if (pos > 0) { numberPart = versionString.substring(0, pos); } - version = Integer.parseInt(numberPart.replaceAll("\\.", "")); + + String versions[] = numberPart.split("\\."); + int major = Integer.parseInt(versions[0]); + int minor = Integer.parseInt(versions[1]); + int patch = Integer.parseInt(versions[2]); + // version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602) + version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch)); } catch (Exception e) { logger.error("Can not recognize Spark version " + versionString + ". Assume it's a future release", e); // assume it is future release - version = 999; + version = 99999; } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/resources/R/zeppelin_sparkr.R ---------------------------------------------------------------------- diff --git a/spark/src/main/resources/R/zeppelin_sparkr.R b/spark/src/main/resources/R/zeppelin_sparkr.R index d951774..8e66421 100644 --- a/spark/src/main/resources/R/zeppelin_sparkr.R +++ b/spark/src/main/resources/R/zeppelin_sparkr.R @@ -42,7 +42,7 @@ assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv) # setup spark env assign(".sc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkContext"), envir = SparkR:::.sparkREnv) assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv) -if (version >= 200) { +if (version >= 20000) { assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkSession"), envir = SparkR:::.sparkREnv) assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv) } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/resources/python/zeppelin_pyspark.py ---------------------------------------------------------------------- diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/src/main/resources/python/zeppelin_pyspark.py index 9a40556..3e6535f 100644 --- a/spark/src/main/resources/python/zeppelin_pyspark.py +++ b/spark/src/main/resources/python/zeppelin_pyspark.py @@ -107,9 +107,9 @@ class PyZeppelinContext(dict): class SparkVersion(object): - SPARK_1_4_0 = 140 - SPARK_1_3_0 = 130 - SPARK_2_0_0 = 200 + SPARK_1_4_0 = 10400 + SPARK_1_3_0 = 10300 + SPARK_2_0_0 = 20000 def __init__(self, versionNumber): self.version = versionNumber http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java ---------------------------------------------------------------------- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java index 5783c1e..3dc8f4e 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java @@ -24,7 +24,7 @@ public class SparkVersionTest { @Test public void testUnknownSparkVersion() { - assertEquals(999, SparkVersion.fromVersionString("DEV-10.10").toNumber()); + assertEquals(99999, SparkVersion.fromVersionString("DEV-10.10").toNumber()); } @Test @@ -33,6 +33,8 @@ public class SparkVersionTest { assertFalse(SparkVersion.fromVersionString("1.5.9").isUnsupportedVersion()); assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion()); assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion()); + // should support spark2 version of HDP 2.5 + assertFalse(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion()); } @Test @@ -40,6 +42,9 @@ public class SparkVersionTest { // test equals assertEquals(SparkVersion.SPARK_1_2_0, SparkVersion.fromVersionString("1.2.0")); assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT")); + assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT")); + // test spark2 version of HDP 2.5 + assertEquals(SparkVersion.SPARK_2_0_0, SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245")); // test newer than assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_2_0)); @@ -60,7 +65,7 @@ public class SparkVersionTest { assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0)); // conversion - assertEquals(120, SparkVersion.SPARK_1_2_0.toNumber()); + assertEquals(10200, SparkVersion.SPARK_1_2_0.toNumber()); assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString()); } }
