zeppelin git commit: ZEPPELIN-2224. Throw more meaningful exception when kerberos is enabled in livy interpreter
e.execute(RestTemplate.java:580) at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:498) at org.apache.zeppelin.livy.BaseLivyInterprereter.callRestAPI(BaseLivyInterprereter.java:407) at org.apache.zeppelin.livy.BaseLivyInterprereter.createSession(BaseLivyInterprereter.java:193) at org.apache.zeppelin.livy.BaseLivyInterprereter.initLivySession(BaseLivyInterprereter.java:99) ``` ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2104 from zjffdu/ZEPPELIN-2224 and squashes the following commits: 67feb4c [Jeff Zhang] code cleanup 0221193 [Jeff Zhang] ZEPPELIN-2224. Throw more meaningful exception when kerberos is enabled in livy interpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d5b46987 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d5b46987 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d5b46987 Branch: refs/heads/master Commit: d5b46987f0e513bbaa20c681be01df8a32ae2674 Parents: 61b7162 Author: Jeff Zhang Authored: Tue Mar 7 15:38:29 2017 +0800 Committer: Jeff Zhang Committed: Thu Apr 6 14:47:45 2017 +0800 -- .../java/org/apache/zeppelin/livy/BaseLivyInterprereter.java| 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d5b46987/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index 26342a2..8fd0648 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -25,6 +25,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; import org.apache.http.impl.client.HttpClients; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -369,13 +370,11 @@ public abstract class BaseLivyInterprereter extends Interpreter { } if (displayAppInfo) { -//TODO(zjffdu), use multiple InterpreterResult to display appInfo InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); interpreterResult.add(InterpreterResult.Type.TEXT, result); String appInfoHtml = "Spark Application Id: " + sessionInfo.appId + "" + "Spark WebUI: " + sessionInfo.webUIAddress + ""; -LOGGER.info("appInfoHtml:" + appInfoHtml); interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml); return interpreterResult; } else { @@ -486,6 +485,8 @@ public abstract class BaseLivyInterprereter extends Interpreter { if (cause.getResponseBodyAsString().matches(SESSION_NOT_FOUND_PATTERN)) { throw new SessionNotFoundException(cause.getResponseBodyAsString()); } +throw new LivyException(cause.getResponseBodyAsString() + "\n" ++ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e))); } throw new LivyException(e); }
zeppelin git commit: ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master
Repository: zeppelin Updated Branches: refs/heads/master 5381883ff -> 495be1ede ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master ### What is this PR for? `SparkUI.addUIAddress` is removed in spark master which cause this error. Actually spark 2.0 introduce new api `SparkContext.uiWebUrl` (SPARK-14576) which is public in contrast `SparkUI.addUIAddress` is private. This PR would use `SparkUI.addUIAddress` for spark 1 and `SparkContext.uiWebUrl` for spark 2. ### What type of PR is it? [Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2150 ### How should this be tested? Manually verify it on spark 1.6.2, spark 2.1.0 and spark master ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2231 from zjffdu/ZEPPELIN-2150 and squashes the following commits: 499b623 [Jeff Zhang] return null when sparkUIUrl is not avaialbe e285278 [Jeff Zhang] ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/495be1ed Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/495be1ed Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/495be1ed Branch: refs/heads/master Commit: 495be1ede1c0f5fc949275153dcaa1c6de5ddcdd Parents: 5381883 Author: Jeff Zhang Authored: Sat Apr 8 08:15:14 2017 +0800 Committer: Jeff Zhang Committed: Sat Apr 8 16:55:46 2017 +0800 -- .../apache/zeppelin/spark/SparkInterpreter.java| 17 + 1 file changed, 13 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/495be1ed/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 6a2f3cc..af97d87 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -999,10 +999,19 @@ public class SparkInterpreter extends Interpreter { if (sparkUrl != null) { return sparkUrl; } -Option sparkUiOption = (Option) Utils.invokeMethod(sc, "ui"); -SparkUI sparkUi = sparkUiOption.get(); -String sparkWebUrl = sparkUi.appUIAddress(); -return sparkWebUrl; + +if (sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0)) { + Option uiWebUrlOption = (Option) Utils.invokeMethod(sc, "uiWebUrl"); + if (uiWebUrlOption.isDefined()) { +return uiWebUrlOption.get(); + } +} else { + Option sparkUIOption = (Option) Utils.invokeMethod(sc, "ui"); + if (sparkUIOption.isDefined()) { +return (String) Utils.invokeMethod(sparkUIOption.get(), "appUIAddress"); + } +} +return null; } private Results.Result interpret(String line) {
zeppelin git commit: ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 d0b4287d5 -> 46b196575 ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master `SparkUI.addUIAddress` is removed in spark master which cause this error. Actually spark 2.0 introduce new api `SparkContext.uiWebUrl` (SPARK-14576) which is public in contrast `SparkUI.addUIAddress` is private. This PR would use `SparkUI.addUIAddress` for spark 1 and `SparkContext.uiWebUrl` for spark 2. [Improvement | Refactoring] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-2150 Manually verify it on spark 1.6.2, spark 2.1.0 and spark master * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2231 from zjffdu/ZEPPELIN-2150 and squashes the following commits: 499b623 [Jeff Zhang] return null when sparkUIUrl is not avaialbe e285278 [Jeff Zhang] ZEPPELIN-2150. NoSuchMethodError: org.apache.spark.ui.SparkUI.appUIAddress() for the spark master (cherry picked from commit 495be1ede1c0f5fc949275153dcaa1c6de5ddcdd) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/46b19657 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/46b19657 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/46b19657 Branch: refs/heads/branch-0.7 Commit: 46b1965752522f255e34002dbd9288cca1495b44 Parents: d0b4287 Author: Jeff Zhang Authored: Sat Apr 8 08:15:14 2017 +0800 Committer: Jeff Zhang Committed: Sat Apr 8 18:12:33 2017 +0800 -- .../apache/zeppelin/spark/SparkInterpreter.java | 22 +++- 1 file changed, 17 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/46b19657/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 47f8080..a2eeac6 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -961,11 +961,23 @@ public class SparkInterpreter extends Interpreter { numReferenceOfSparkContext.incrementAndGet(); } - private String getSparkUIUrl() { -Option sparkUiOption = (Option) Utils.invokeMethod(sc, "ui"); -SparkUI sparkUi = sparkUiOption.get(); -String sparkWebUrl = sparkUi.appUIAddress(); -return sparkWebUrl; + public String getSparkUIUrl() { +if (sparkUrl != null) { + return sparkUrl; +} + +if (sparkVersion.newerThanEquals(SparkVersion.SPARK_2_0_0)) { + Option uiWebUrlOption = (Option) Utils.invokeMethod(sc, "uiWebUrl"); + if (uiWebUrlOption.isDefined()) { +return uiWebUrlOption.get(); + } +} else { + Option sparkUIOption = (Option) Utils.invokeMethod(sc, "ui"); + if (sparkUIOption.isDefined()) { +return (String) Utils.invokeMethod(sparkUIOption.get(), "appUIAddress"); + } +} +return null; } private Results.Result interpret(String line) {
zeppelin git commit: [ZEPPELIN-2355] Livy cancel enhancements
Repository: zeppelin Updated Branches: refs/heads/master e5922b6bb -> 861f1d88f [ZEPPELIN-2355] Livy cancel enhancements ### What is this PR for? The Cancel functionality for the Livy interpreter has few issues. One issue is because a variable is not published correctly. Second issue is observed when there is a delay in launching the application. Any cancel before application launch is ignored. The third issue is that Cancel is not correctly implemented for SparkSQLInterpreter. ### What type of PR is it? Bug Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2355 ### How should this be tested? The test cases are modified to test the changes. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Benoy Antony Closes #2223 from benoyantony/livy-cancel-enhancement and squashes the following commits: 244e6d3 [Benoy Antony] clear the cancel requests if livy doesnt't support cancellation and modified testcase 75fe574 [Benoy Antony] added testcase for cancellation support on LivySparkSQLInterpreter and moved the removal to finally block 9fc6dbf [Benoy Antony] remove unrelated changes in imports 8673acf [Benoy Antony] ZEPPELIN-2355 Fix race conditions while cancelling a paragraph Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/861f1d88 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/861f1d88 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/861f1d88 Branch: refs/heads/master Commit: 861f1d88fe2ea105df6892abef14142327c49f6f Parents: e5922b6 Author: Benoy Antony Authored: Tue Apr 11 20:55:36 2017 -0700 Committer: Jeff Zhang Committed: Mon Apr 17 09:16:37 2017 +0800 -- .../zeppelin/livy/BaseLivyInterprereter.java| 52 ++- .../zeppelin/livy/LivySparkSQLInterpreter.java | 5 ++ .../apache/zeppelin/livy/LivyInterpreterIT.java | 94 ++-- 3 files changed, 123 insertions(+), 28 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/861f1d88/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index 8fd0648..43cd507 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -43,13 +43,17 @@ import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.ConcurrentHashMap; + + /** * Base class for livy interpreters. */ @@ -68,9 +72,8 @@ public abstract class BaseLivyInterprereter extends Interpreter { protected LivyVersion livyVersion; private RestTemplate restTemplate; - // keep tracking the mapping between paragraphId and statementId, so that we can cancel the - // statement after we execute it. - private ConcurrentHashMap paragraphId2StmtIdMap = new ConcurrentHashMap<>(); + Set paragraphsToCancel = Collections.newSetFromMap( + new ConcurrentHashMap()); private ConcurrentHashMap paragraphId2StmtProgressMap = new ConcurrentHashMap<>(); @@ -163,21 +166,8 @@ public abstract class BaseLivyInterprereter extends Interpreter { @Override public void cancel(InterpreterContext context) { -if (livyVersion.isCancelSupported()) { - String paraId = context.getParagraphId(); - Integer stmtId = paragraphId2StmtIdMap.get(paraId); - try { -if (stmtId != null) { - cancelStatement(stmtId); -} - } catch (LivyException e) { -LOGGER.error("Fail to cancel statement " + stmtId + " for paragraph " + paraId, e); - } finally { -paragraphId2StmtIdMap.remove(paraId); - } -} else { - LOGGER.warn("cancel is not supported for this version of livy: " + livyVersion); -} +paragraphsToCancel.add(context.getParagraphId()); +LOGGER.info("Added paragraph " + context.getParagraphId() + " for cancellation."); } @Override @@ -261,11 +251,12 @@ public abstract class BaseLivyInterprereter extends Interpreter { } stmtInfo = executeStatement(new ExecuteRequest(code)); } - if (paragraphId != null) { -paragraphId2StmtIdMap.put(paragraphId, stmtInfo.id); - } // pull th
zeppelin git commit: ZEPPELIN-2407. Livy Interpreter always return plain text result
Repository: zeppelin Updated Branches: refs/heads/master 861f1d88f -> 5fd4ffc56 ZEPPELIN-2407. Livy Interpreter always return plain text result ### What is this PR for? It happens when zeppelin.livy.displayAppInfo is true. Straightforward fix. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2407 ### How should this be tested? Test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2254 from zjffdu/ZEPPELIN-2407 and squashes the following commits: ba59a77 [Jeff Zhang] ZEPPELIN-2407. Livy Interpreter always return plain text result Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5fd4ffc5 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5fd4ffc5 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5fd4ffc5 Branch: refs/heads/master Commit: 5fd4ffc56658884b8f671ca40b1f78b8354f1f69 Parents: 861f1d8 Author: Jeff Zhang Authored: Sun Apr 16 16:38:36 2017 +0800 Committer: Jeff Zhang Committed: Mon Apr 17 09:18:38 2017 +0800 -- .../apache/zeppelin/livy/BaseLivyInterprereter.java | 2 +- .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 16 +++- 2 files changed, 16 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5fd4ffc5/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index 43cd507..77c98d9 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -379,7 +379,7 @@ public abstract class BaseLivyInterprereter extends Interpreter { if (displayAppInfo) { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); -interpreterResult.add(InterpreterResult.Type.TEXT, result); +interpreterResult.add(result); String appInfoHtml = "Spark Application Id: " + sessionInfo.appId + "" + "Spark WebUI: " + sessionInfo.webUIAddress + ""; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5fd4ffc5/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java -- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 6537125..06da238 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -145,6 +145,13 @@ public class LivyInterpreterIT { assertTrue(result.message().get(0).getData().contains("defined object Person")); } + // html output + String htmlCode = "println(\"%html hello \")"; + result = sparkInterpreter.interpret(htmlCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + // error result = sparkInterpreter.interpret("println(a)", context); assertEquals(InterpreterResult.Code.ERROR, result.code()); @@ -626,8 +633,15 @@ public class LivyInterpreterIT { InterpreterResult result = sparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(2, result.message().size()); - assertTrue(result.message().get(1).getData().contains("Spark Application Id")); + + // html output + String htmlCode = "println(\"%html hello \")"; + result = sparkInterpreter.interpret(htmlCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(2, result.message().size()); + assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + } finally { sparkInterpreter.close(); }
zeppelin git commit: ZEPPELIN-2377. Hive Support can not be enabled in spark master
Repository: zeppelin Updated Branches: refs/heads/master 5fd4ffc56 -> 74c0408d3 ZEPPELIN-2377. Hive Support can not be enabled in spark master ### What is this PR for? The root cause is that `org.apache.spark.sql.hive.HiveSessionState` is removed in spark master. I change it to `org.apache.spark.sql.hive.execution.InsertIntoHiveTable` which is existed early in spark 1.0. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2377 ### How should this be tested? Verify it manually in spark master, spark 2.1.0 and spark 1.6.2 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2237 from zjffdu/ZEPPELIN-2377 and squashes the following commits: d16196b [Jeff Zhang] ZEPPELIN-2377. Hive Support can not be enabled in spark master Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/74c0408d Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/74c0408d Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/74c0408d Branch: refs/heads/master Commit: 74c0408d30927b11c99a8e134284be4156b19f8a Parents: 5fd4ffc Author: Jeff Zhang Authored: Sun Apr 9 15:46:32 2017 +0800 Committer: Jeff Zhang Committed: Mon Apr 17 09:21:32 2017 +0800 -- .../src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/74c0408d/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index af97d87..49c83dc 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -246,7 +246,7 @@ public class SparkInterpreter extends Interpreter { */ private boolean hiveClassesArePresent() { try { - this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState"); + this.getClass().forName("org.apache.spark.sql.hive.execution.InsertIntoHiveTable"); this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf"); return true; } catch (ClassNotFoundException | NoClassDefFoundError e) {
zeppelin git commit: ZEPPELIN-2390. Improve returnType for z.checkbox
Repository: zeppelin Updated Branches: refs/heads/master 33663941c -> a7ffc1291 ZEPPELIN-2390. Improve returnType for z.checkbox ### What is this PR for? Currently it is not convenient to access the individual item of the return value of z.checkbox, I would propose to return `Seq` for `SparkInterpreter` and `list` for `PySparkInterpreter`. This might cause some incompatibility, but I think it is acceptable considering the benefits. Besides that, before this PR, all the items of checkbox would be checked by default in `PySparkInterpreter` which is inconsistent with `SparkInterpreter`, so I change it to nothing is selected by default in this PR. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2390 ### How should this be tested? Unit test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2241 from zjffdu/ZEPPELIN-2390 and squashes the following commits: 75e3afc [Jeff Zhang] ZEPPELIN-2390. Improve returnType for z.checkbox Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/a7ffc129 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/a7ffc129 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/a7ffc129 Branch: refs/heads/master Commit: a7ffc1291885cc555ed84bd71f455ebcb2e9dd62 Parents: 3366394 Author: Jeff Zhang Authored: Tue Apr 11 18:09:35 2017 +0800 Committer: Jeff Zhang Committed: Tue Apr 18 17:52:27 2017 +0800 -- .../apache/zeppelin/spark/ZeppelinContext.java | 9 ++-- .../main/resources/python/zeppelin_pyspark.py | 10 ++-- .../java/org/apache/zeppelin/display/GUI.java | 4 +- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 54 +--- 4 files changed, 61 insertions(+), 16 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a7ffc129/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index 6e96d9d..e187915 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -136,7 +136,7 @@ public class ZeppelinContext { } @ZeppelinApi - public scala.collection.Iterable checkbox(String name, + public scala.collection.Seq checkbox(String name, scala.collection.Iterable> options) { List allChecked = new LinkedList<>(); for (Tuple2 option : asJavaIterable(options)) { @@ -146,11 +146,12 @@ public class ZeppelinContext { } @ZeppelinApi - public scala.collection.Iterable checkbox(String name, + public scala.collection.Seq checkbox(String name, scala.collection.Iterable defaultChecked, scala.collection.Iterable> options) { -return collectionAsScalaIterable(gui.checkbox(name, asJavaCollection(defaultChecked), - tuplesToParamOptions(options))); +return scala.collection.JavaConversions.asScalaBuffer( +gui.checkbox(name, asJavaCollection(defaultChecked), +tuplesToParamOptions(options))).toSeq(); } private ParamOption[] tuplesToParamOptions( http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a7ffc129/spark/src/main/resources/python/zeppelin_pyspark.py -- diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/src/main/resources/python/zeppelin_pyspark.py index 5029d59..da4d794 100644 --- a/spark/src/main/resources/python/zeppelin_pyspark.py +++ b/spark/src/main/resources/python/zeppelin_pyspark.py @@ -97,13 +97,15 @@ class PyZeppelinContext(dict): def checkbox(self, name, options, defaultChecked=None): if defaultChecked is None: - defaultChecked = list(map(lambda items: items[0], options)) + defaultChecked = [] optionTuples = list(map(lambda items: self.__tupleToScalaTuple2(items), options)) optionIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(optionTuples) defaultCheckedIterables = gateway.jvm.scala.collection.JavaConversions.collectionAsScalaIterable(defaultChecked) - -checkedIterables = self.z.checkbox(name, defaultCheckedIterables, optionIterables) -return gateway.jvm.scala.collection.JavaConversions.asJavaCollection(checkedIterables) +checkedItems = gateway.jvm.scala.collection.JavaConversions.seqAsJavaList(self.z.checkbox(name, defaultCheckedIterables, optionIterables)) +result = [] +for
zeppelin git commit: ZEPPELIN-2386. Add parameter for check running current paragraph in ZeppelinContext
Repository: zeppelin Updated Branches: refs/heads/master a7ffc1291 -> c706d453e ZEPPELIN-2386. Add parameter for check running current paragraph in ZeppelinContext ### What is this PR for? I can reproduce it via the test code in ZeppelinIT.testAngularDisplay. First run the 4 paragraphs to initiate the state. Then when I click paragraph 0 the first time, paragraph 2 will run correctly. But if I click paragraph 0 again, interpreter would raise the following exception. The cause is that ZeppelinContext's interpreterContext now point to paragraph 2, because paragraph 2 is the last paragraph that runs in this interpreter. So I propose to add parameter for check running current paragraph in ZeppelinContext. * Add 2 methods in ZeppelinContext. `run(idx, checkCurrentParagraph) `, 'run(noteId, idx, context, checkCurrentParagraph)' * Also make some changes for debug logging. ### What type of PR is it? [Bug Fix | Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2386 ### How should this be tested? Tested manually and also add integration test. ### Screenshots (if appropriate) Before ![zeppelin_before](https://cloud.githubusercontent.com/assets/164491/24895529/8514826e-1ec3-11e7-915b-70baf09c297a.gif) After ![zeppelin_after](https://cloud.githubusercontent.com/assets/164491/24895532/88e87bc0-1ec3-11e7-90ee-a7ca7a3758f8.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2240 from zjffdu/ZEPPELIN-2386 and squashes the following commits: 03995a3 [Jeff Zhang] ZEPPELIN-2386. Add parameter to check running current paragraph in ZeppelinContext Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c706d453 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c706d453 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c706d453 Branch: refs/heads/master Commit: c706d453e06a2f8c71fcd61d7e4dddc87cafc0f4 Parents: a7ffc12 Author: Jeff Zhang Authored: Tue Apr 11 14:21:53 2017 +0800 Committer: Jeff Zhang Committed: Tue Apr 18 17:56:39 2017 +0800 -- .../apache/zeppelin/spark/ZeppelinContext.java | 60 +--- .../apache/zeppelin/display/AngularObject.java | 4 +- .../apache/zeppelin/socket/NotebookServer.java | 9 +-- .../apache/zeppelin/integration/ZeppelinIT.java | 17 +- 4 files changed, 77 insertions(+), 13 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c706d453/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java index e187915..7e1ab70 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/ZeppelinContext.java @@ -312,7 +312,7 @@ public class ZeppelinContext { */ @ZeppelinApi public void run(String noteId, String paragraphId) { -run(noteId, paragraphId, interpreterContext); +run(noteId, paragraphId, interpreterContext, true); } /** @@ -321,18 +321,38 @@ public class ZeppelinContext { */ @ZeppelinApi public void run(String paragraphId) { +run(paragraphId, true); + } + + /** + * Run paragraph by id + * @param paragraphId + * @param checkCurrentParagraph + */ + @ZeppelinApi + public void run(String paragraphId, boolean checkCurrentParagraph) { String noteId = interpreterContext.getNoteId(); -run(noteId, paragraphId, interpreterContext); +run(noteId, paragraphId, interpreterContext, checkCurrentParagraph); } /** * Run paragraph by id * @param noteId - * @param context */ @ZeppelinApi public void run(String noteId, String paragraphId, InterpreterContext context) { -if (paragraphId.equals(context.getParagraphId())) { +run(noteId, paragraphId, context, true); + } + + /** + * Run paragraph by id + * @param noteId + * @param context + */ + @ZeppelinApi + public void run(String noteId, String paragraphId, InterpreterContext context, + boolean checkCurrentParagraph) { +if (paragraphId.equals(context.getParagraphId()) && checkCurrentParagraph) { throw new InterpreterException("Can not run current Paragraph"); } @@ -412,24 +432,50 @@ public class ZeppelinContext { */ @ZeppelinApi public void run(int idx) { +run(idx, true); + } + + /** + * + * @param idx paragraph index + * @param checkCurrentParagraph check whether you call this run method in th
zeppelin git commit: ZEPPELIN-2395. Refactor Input.java to make dynamic forms extensible
Repository: zeppelin Updated Branches: refs/heads/master 4b1b521fc -> 7b585c739 ZEPPELIN-2395. Refactor Input.java to make dynamic forms extensible ### What is this PR for? Currently, zeppelin only support 3 kinds of dynamic form controls: TextBox, Select, CheckBox. All the things are in `Input.java`, this is hard to add new controls, this PR is for refactoring Input to make dynamic forms extensible. Main Changes: * Make `Input` as the base class of dynamic forms also use it as the factory class * All the concret dynamic forms extend `Input` * Add method `toJson` and `fromJson` for `GUI` for `GUI`'s serialization/deserialization. I plan to do it for other classes as well, so that we can remove duplicated serde code and also make it easy to test serialization/deserialization * Change `z.input` to `z.textbox` as I think z.input is a little misleading. But I still keep `z.input` and make `z.input` as deprecated. * Ideally the new input forms' json should be the same as the old input form json. But there's one bug in the old input form, `type` is missing if the input forms are created in frontend for textbox and select. So I keep the old input forms for compatibility. I will load the old input forms json and convert it into new input forms, and after saving, `note.json` would have the new input forms json. After this PR, user needs to do 3 things to add new ui controls * Implement its UI control classes, (refer TextBox/CheckBox/Select), and specify it in `TypeAdapterFactory` of `Input` for serde. * Add parsing logic in `Input.getInputForm` if you want to support this control in frontend. * Add display logic in `paragraph-parameterizedQueryForm.html` ### What type of PR is it? [ Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2395 ### How should this be tested? Test is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? Yes * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2245 from zjffdu/ZEPPELIN-2395 and squashes the following commits: 16d42a8 [Jeff Zhang] ZEPPELIN-2395. Refactor Input.java to make dynamic forms extensible Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/7b585c73 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/7b585c73 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/7b585c73 Branch: refs/heads/master Commit: 7b585c7399f29492c308f83e342cac29b0c7ca07 Parents: 4b1b521 Author: Jeff Zhang Authored: Wed Apr 12 10:50:16 2017 +0800 Committer: Jeff Zhang Committed: Thu Apr 20 10:56:54 2017 +0800 -- LICENSE | 1 + .../zeppelin/cassandra/InterpreterLogic.scala | 2 +- .../cassandra/InterpreterLogicTest.java | 2 +- .../org/apache/zeppelin/groovy/GObject.java | 2 +- pom.xml | 7 + .../main/resources/python/zeppelin_python.py| 2 +- .../apache/zeppelin/spark/ZeppelinContext.java | 25 ++- zeppelin-interpreter/pom.xml| 5 + .../java/org/apache/zeppelin/display/GUI.java | 69 ++- .../java/org/apache/zeppelin/display/Input.java | 178 +++ .../org/apache/zeppelin/display/OldInput.java | 87 + .../display/RuntimeTypeAdapterFactory.java | 149 .../apache/zeppelin/display/ui/CheckBox.java| 45 + .../apache/zeppelin/display/ui/OptionInput.java | 85 + .../org/apache/zeppelin/display/ui/Select.java | 36 .../org/apache/zeppelin/display/ui/TextBox.java | 38 .../interpreter/remote/RemoteInterpreter.java | 4 +- .../remote/RemoteInterpreterServer.java | 4 +- .../org/apache/zeppelin/display/GUITest.java| 120 + .../org/apache/zeppelin/display/InputTest.java | 35 ++-- .../apache/zeppelin/socket/NotebookServer.java | 5 +- .../integration/ParagraphActionsIT.java | 2 +- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 2 +- .../paragraph-parameterizedQueryForm.html | 20 +-- .../websocketEvents/websocketEvents.factory.js | 1 + .../java/org/apache/zeppelin/notebook/Note.java | 19 ++ .../notebook/repo/AzureNotebookRepo.java| 2 +- .../zeppelin/notebook/repo/S3NotebookRepo.java | 2 +- .../zeppelin/notebook/repo/VFSNotebookRepo.java | 2 +- 29 files changed, 793 insertions(+), 158 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/7b585c73/LICENSE -- diff --git a/LICENSE b/LICENSE index 83a9131..e206a6c 100644 --- a/LICENSE +++ b/LICENSE @@ -255,6 +255,7 @@ The text of ea
zeppelin git commit: [HOTFIX] Fix compilation issue
Repository: zeppelin Updated Branches: refs/heads/master 19b0f30f8 -> 8dd1fb184 [HOTFIX] Fix compilation issue ### What is this PR for? Fix the compilation issue due to #2233 ### What type of PR is it? [Hot Fix] ### Todos * [ ] - Task ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2266 from zjffdu/hotfix and squashes the following commits: 43e7cd9 [Jeff Zhang] [HOTFIX] Fix compilation issue Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/8dd1fb18 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/8dd1fb18 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/8dd1fb18 Branch: refs/heads/master Commit: 8dd1fb184579621133f3212b07b074ee42271bb7 Parents: 19b0f30 Author: Jeff Zhang Authored: Thu Apr 20 13:20:13 2017 +0800 Committer: Jeff Zhang Committed: Thu Apr 20 13:56:42 2017 +0800 -- .../java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java| 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8dd1fb18/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java -- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index ed84caf..ec0e0bd 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -519,7 +519,7 @@ public class ZeppelinSparkClusterTest extends AbstractTestRestApi { @Test public void testPySparkZeppelinContextDynamicForms() throws IOException { Note note = ZeppelinServer.notebook.createNote(anonymous); -Paragraph p = note.addParagraph(AuthenticationInfo.ANONYMOUS); +Paragraph p = note.addNewParagraph(AuthenticationInfo.ANONYMOUS); note.setName("note"); Map config = p.getConfig(); config.put("enabled", true);
zeppelin git commit: ZEPPELIN-2421. AngularObject miss AngularObjectListener when it is pushed from frontend
Repository: zeppelin Updated Branches: refs/heads/master 7a0cb6c7a -> 09e8f7f0c ZEPPELIN-2421. AngularObject miss AngularObjectListener when it is pushed from frontend ### What is this PR for? This bug happens when you save your note, and restart zeppelin and run it again. The root cause is that the angular object miss AngularObjectListener which means you can not sync up the changes on angular object to frontend. This PR would restore the listener correctly when we restore the angular objects saved before. Also make some changes for logging in this PR. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2421 ### How should this be tested? Tested manually ### Screenshots (if appropriate) Before ![zeppelin_before](https://cloud.githubusercontent.com/assets/164491/25173268/0824dc72-2526-11e7-955c-ea5e20dae746.gif) After ![zeppelin_after](https://cloud.githubusercontent.com/assets/164491/25173272/09e11bfc-2526-11e7-9096-5adb6f1992ef.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2263 from zjffdu/ZEPPELIn-2421 and squashes the following commits: c2503a4 [Jeff Zhang] ZEPPELIN-2421. AngularObject miss AngularObjectListener when it is pushed from frontend Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/09e8f7f0 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/09e8f7f0 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/09e8f7f0 Branch: refs/heads/master Commit: 09e8f7f0cd84efd9c67fff44119f7953881d91ab Parents: 7a0cb6c Author: Jeff Zhang Authored: Wed Apr 19 17:26:21 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 21 11:05:04 2017 +0800 -- .../java/org/apache/zeppelin/display/AngularObjectRegistry.java | 5 + .../interpreter/remote/RemoteInterpreterEventClient.java| 5 +++-- .../interpreter/remote/RemoteInterpreterEventPoller.java| 3 +++ .../main/java/org/apache/zeppelin/socket/NotebookServer.java| 4 ++-- 4 files changed, 13 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/09e8f7f0/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java -- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java index 6b2bfac..a993992 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/display/AngularObjectRegistry.java @@ -252,5 +252,10 @@ public class AngularObjectRegistry { public void setRegistry(Map> registry) { this.registry = registry; +for (Map map : registry.values()) { + for (AngularObject ao : map.values()) { +ao.setListener(angularObjectListener); + } +} } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/09e8f7f0/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java -- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java index 73c14f1..bb6de31 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventClient.java @@ -46,7 +46,7 @@ import java.util.Map; * RemoteInterpreterEventPoller is counter part in ZeppelinServer */ public class RemoteInterpreterEventClient implements ResourcePoolConnector { - private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEvent.class); + private final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventClient.class); private final List eventQueue = new LinkedList<>(); private final List getAllResourceResponse = new LinkedList<>(); private final Map getResourceResponse = new HashMap<>(); @@ -415,6 +415,7 @@ public class RemoteInterpreterEventClient implements ResourcePoolConnector { } private void sendEvent(RemoteInterpreterEvent event) { +logger.debug("Send Event: " + event); synchronized (eventQueue) { eventQueue.add(event); eventQueue.notifyAll(); @@ -446,7 +447,7 @@ public class Remot
zeppelin git commit: ZEPPELIN-2445. Display which line of python code raise exception
Repository: zeppelin Updated Branches: refs/heads/master 6eecdecb5 -> 5cd806dc3 ZEPPELIN-2445. Display which line of python code raise exception ### What is this PR for? When python code raise exception, user can only see the stacktrace, but don't know which line cause it. It is not so convenient especially when you write many lines of code. This PR would display which line raise exception. Besides that, I also fix another issue that we would have duplicated error message in python3. (See screenshot) ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2445 ### How should this be tested? Unit test is added ### Screenshots (if appropriate) Python 2 Before ![image](https://cloud.githubusercontent.com/assets/164491/25367168/89ce67fa-29a6-11e7-900c-cff2c66a6df4.png) Python2 After ![image](https://cloud.githubusercontent.com/assets/164491/25366936/65d246d8-29a5-11e7-8ad7-6786252e7913.png) Python 3 Before ![image](https://cloud.githubusercontent.com/assets/164491/25367181/9d9a1608-29a6-11e7-8a62-5404b5941256.png) Python3 After ![image](https://cloud.githubusercontent.com/assets/164491/25366943/6f0200c2-29a5-11e7-8fe6-80794e7c7d86.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2280 from zjffdu/ZEPPELIN-2445 and squashes the following commits: fa0cfba [Jeff Zhang] ZEPPELIN-2445. Display which line of python code raise exception Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5cd806dc Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5cd806dc Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5cd806dc Branch: refs/heads/master Commit: 5cd806dc3290cbcdcfb7db4f160e32077cf2e522 Parents: 6eecdec Author: Jeff Zhang Authored: Mon Apr 24 15:55:14 2017 +0800 Committer: Jeff Zhang Committed: Thu Apr 27 13:10:06 2017 +0800 -- .../zeppelin/spark/PySparkInterpreter.java | 32 ++-- .../main/resources/python/zeppelin_pyspark.py | 51 spark/src/test/resources/log4j.properties | 47 ++ .../util/InterpreterOutputStream.java | 2 +- .../zeppelin/rest/ZeppelinSparkClusterTest.java | 21 .../src/test/resources/log4j.properties | 3 +- 6 files changed, 109 insertions(+), 47 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5cd806dc/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java index bf0a915..56cec94 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java @@ -63,7 +63,7 @@ import py4j.GatewayServer; * */ public class PySparkInterpreter extends Interpreter implements ExecuteResultHandler { - Logger logger = LoggerFactory.getLogger(PySparkInterpreter.class); + private static final Logger LOGGER = LoggerFactory.getLogger(PySparkInterpreter.class); private GatewayServer gatewayServer; private DefaultExecutor executor; private int port; @@ -106,7 +106,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand throw new InterpreterException(e); } -logger.info("File {} created", scriptPath); +LOGGER.info("File {} created", scriptPath); } @Override @@ -131,7 +131,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand try { urlList.add(f.toURI().toURL()); } catch (MalformedURLException e) { - logger.error("Error", e); + LOGGER.error("Error", e); } } } @@ -148,7 +148,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand try { urlList.add(f.toURI().toURL()); } catch (MalformedURLException e) { - logger.error("Error", e); + LOGGER.error("Error", e); } } } @@ -162,7 +162,7 @@ public class PySparkInterpreter extends Interpreter implements ExecuteResultHand Thread.currentThread().setContextClassLoader(newCl); createGatewayServerAndStartScript(); } catch (Exception e) { - logger.error("Error", e); + LOGGER.error("Error", e); throw new InterpreterException(e);
zeppelin git commit: [HOTFIX] Add livy build in .travis
Repository: zeppelin Updated Branches: refs/heads/master 5cd806dc3 -> 0f1701da8 [HOTFIX] Add livy build in .travis ### What is this PR for? livy integration test is ignored, this PR add livy integration test in travis ### What type of PR is it? [Hot Fix] ### Todos * [ ] - Task ### What is the Jira issue? * No jira created ### How should this be tested? CI pass ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2279 from zjffdu/hotfix_livy and squashes the following commits: 674c987 [Jeff Zhang] [HOTFIX] Add livy build in .travis Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/0f1701da Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/0f1701da Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/0f1701da Branch: refs/heads/master Commit: 0f1701da8bb98678ae8e486796b2c1e650125106 Parents: 5cd806d Author: Jeff Zhang Authored: Fri Apr 21 17:38:51 2017 +0800 Committer: Jeff Zhang Committed: Thu Apr 27 16:36:18 2017 +0800 -- .travis.yml | 16 +--- .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 8 ++-- 2 files changed, 15 insertions(+), 9 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0f1701da/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index f77906f..72104db 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,13 +73,15 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" -# Test python/pyspark with python 2 -- jdk: "oraclejdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" - -# Test python/pyspark with python 3 -- jdk: "oraclejdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" +# Test python/pyspark with python 2, livy 0.2 +- sudo: required + jdk: "oraclejdk7" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + +# Test python/pyspark with python 3, livy 0.3 +- sudo: required + jdk: "oraclejdk7" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" before_install: # check files included in commit range, clear bower_component
zeppelin git commit: ZEPPELIN-2079. Upgrade livy to 0.3 in livy interpreter
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 f1064f62d -> c4cf06869 ZEPPELIN-2079. Upgrade livy to 0.3 in livy interpreter Upgrade livy to 0.3. * Add new tests for livy 0.3 * Add 2 livy build in travis. (livy 0.2 + spark 1.6.3 and livy 0.3 + spark 2.1.0, unfortunately livy 0.3 has some packaging issue which cause some issues for integration test for livy 0.3 + spark 1.6.3). I also merge the livy build into spark build in travis but has to set `sudo` as `required` for more memory. [Improvement] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-2079 Tested is added * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2010 from zjffdu/ZEPPELIN-2079 and squashes the following commits: e9d1042 [Jeff Zhang] update travis 2695d7c [Jeff Zhang] ZEPPELIN-2079. Upgrade livy to 0.3 in livy interpreter (cherry picked from commit d55058b05d41ef212ec556b2ce762ccc66e407cb) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c4cf0686 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c4cf0686 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c4cf0686 Branch: refs/heads/branch-0.7 Commit: c4cf0686915867199fc17a228746ce2ae271f4e1 Parents: f1064f6 Author: Jeff Zhang Authored: Mon Feb 13 09:06:39 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 07:44:43 2017 +0800 -- .travis.yml | 3 +- livy/pom.xml| 968 ++- .../zeppelin/livy/BaseLivyInterprereter.java| 2 +- .../zeppelin/livy/LivySparkInterpreter.java | 2 +- .../org/apache/zeppelin/livy/LivyVersion.java | 4 +- .../apache/zeppelin/livy/LivyInterpreterIT.java | 168 +++- 6 files changed, 664 insertions(+), 483 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c4cf0686/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 47d2f94..a5a9acb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,7 +82,8 @@ matrix: env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test livy with spark 1.5.2 and hadoop 2.6 -- jdk: "oraclejdk7" +- sudo: required + jdk: "oraclejdk7" env: SCALA_VER="2.10" $LIVY_VER="0.2.0" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Phadoop-2.6" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl zeppelin-interpreter,livy" TEST_PROJECTS="-DfailIfNoTests=false" before_install: http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c4cf0686/livy/pom.xml -- diff --git a/livy/pom.xml b/livy/pom.xml index 869a2a0..5f41308 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -45,452 +45,536 @@ 3.2.4-Zeppelin 1.7.0 1.9.5 -0.2.0 -1.5.2 +0.3.0 +2.1.0 2.6.0 - 2.16 1.8 - - - ${project.groupId} - zeppelin-interpreter - ${project.version} - provided - - - - org.apache.commons - commons-exec - ${commons.exec.version} - - - - org.slf4j - slf4j-api - - - - org.slf4j - slf4j-log4j12 - - - - org.apache.httpcomponents - httpclient - ${httpcomponents.client.version} - - - - com.google.code.gson - gson - - - - org.springframework.security.kerberos - spring-security-kerberos-client - ${spring.security.kerberosclient} - - - - org.springframework - spring-web - ${spring.web.version} - - - - junit - junit - test - - - - org.assertj - assertj-core - ${assertj.version} - test - - - org.mockito - mockito-all - ${mockito.version} - test - - - - com.cloudera.livy - livy-integration-test - ${livy.version} - test - - - org.xerial.snappy - snappy-java - -
zeppelin git commit: [HOTFIX] Add livy build in .travis
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 c4cf06869 -> b9bc2b810 [HOTFIX] Add livy build in .travis livy integration test is ignored, this PR add livy integration test in travis [Hot Fix] * [ ] - Task * No jira created CI pass * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2279 from zjffdu/hotfix_livy and squashes the following commits: 674c987 [Jeff Zhang] [HOTFIX] Add livy build in .travis (cherry picked from commit 0f1701da8bb98678ae8e486796b2c1e650125106) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/b9bc2b81 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/b9bc2b81 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/b9bc2b81 Branch: refs/heads/branch-0.7 Commit: b9bc2b810dea27fcef44062f56c4446eacb70b64 Parents: c4cf068 Author: Jeff Zhang Authored: Fri Apr 21 17:38:51 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:05:04 2017 +0800 -- .travis.yml | 15 ++- .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 8 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b9bc2b81/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index a5a9acb..c7e97d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,18 +73,15 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.* -DfailIfNoTests=false" -# Test python/pyspark with python 2 -- jdk: "oraclejdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Ppyspark" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" - -# Test python/pyspark with python 3 -- jdk: "oraclejdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Ppyspark -Pscala-2.11" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" +# Test python/pyspark with python 2, livy 0.2 +- sudo: required + jdk: "oraclejdk7" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" -# Test livy with spark 1.5.2 and hadoop 2.6 +# Test python/pyspark with python 3, livy 0.3 - sudo: required jdk: "oraclejdk7" - env: SCALA_VER="2.10" $LIVY_VER="0.2.0" SPARK_VER="1.5.2" HADOOP_VER="2.6" PROFILE="-Pspark-1.5 -Phadoop-2.6" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl zeppelin-interpreter,livy" TEST_PROJECTS="-DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="package -am -DskipTests -DskipRat" TEST_FLAG="
zeppelin git commit: ZEPPELIN-1595. Make ZeppelinContext extensible
Repository: zeppelin Updated Branches: refs/heads/master 0f1701da8 -> c8cd1cf50 ZEPPELIN-1595. Make ZeppelinContext extensible ### What is this PR for? For now, ZeppelinContext only support Spark Interpreter. I'd like to make it extensible, so that it can support other interpreters as well. For now, user need to implement the following 3 methods to extend ZeppelinContext * public List getSupportedClasses() * public abstract Map getInterpreterClassMap(); * protected abstract String showData(Object obj); ### What type of PR is it? [Feature | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1595 ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #1574 from zjffdu/ZEPPELIN-1595 and squashes the following commits: 754c1e0 [Jeff Zhang] ZEPPELIN-1595. Make ZeppelinContext extensible Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c8cd1cf5 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c8cd1cf5 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c8cd1cf5 Branch: refs/heads/master Commit: c8cd1cf5065605269cf83ebc8fe7b265337e34a2 Parents: 0f1701d Author: Jeff Zhang Authored: Mon Oct 31 15:33:51 2016 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:23:34 2017 +0800 -- .../apache/zeppelin/rinterpreter/RStatics.java |6 +- .../apache/zeppelin/rinterpreter/RContext.scala |6 +- .../zeppelin/spark/PySparkInterpreter.java |4 +- .../apache/zeppelin/spark/SparkInterpreter.java | 19 +- .../zeppelin/spark/SparkSqlInterpreter.java |2 +- .../zeppelin/spark/SparkZeppelinContext.java| 280 + .../apache/zeppelin/spark/ZeppelinContext.java | 1019 -- .../apache/zeppelin/spark/ZeppelinRContext.java |6 +- .../main/resources/python/zeppelin_pyspark.py |2 +- .../interpreter/BaseZeppelinContext.java| 821 ++ 10 files changed, 1120 insertions(+), 1045 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c8cd1cf5/r/src/main/java/org/apache/zeppelin/rinterpreter/RStatics.java -- diff --git a/r/src/main/java/org/apache/zeppelin/rinterpreter/RStatics.java b/r/src/main/java/org/apache/zeppelin/rinterpreter/RStatics.java index 361fe47..1ea35ce 100644 --- a/r/src/main/java/org/apache/zeppelin/rinterpreter/RStatics.java +++ b/r/src/main/java/org/apache/zeppelin/rinterpreter/RStatics.java @@ -25,7 +25,7 @@ package org.apache.zeppelin.rinterpreter; import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SQLContext; -import org.apache.zeppelin.spark.ZeppelinContext; +import org.apache.zeppelin.spark.SparkZeppelinContext; /** * RStatics provides static class methods that can be accessed through the SparkR bridge @@ -33,7 +33,7 @@ import org.apache.zeppelin.spark.ZeppelinContext; */ public class RStatics { private static SparkContext sc = null; - private static ZeppelinContext z = null; + private static SparkZeppelinContext z = null; private static SQLContext sql = null; private static RContext rCon = null; @@ -42,7 +42,7 @@ public class RStatics { return sc; } - public static ZeppelinContext setZ(ZeppelinContext newZ) { + public static SparkZeppelinContext setZ(SparkZeppelinContext newZ) { z = newZ; return z; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c8cd1cf5/r/src/main/scala/org/apache/zeppelin/rinterpreter/RContext.scala -- diff --git a/r/src/main/scala/org/apache/zeppelin/rinterpreter/RContext.scala b/r/src/main/scala/org/apache/zeppelin/rinterpreter/RContext.scala index ffab160..39ba5ae 100644 --- a/r/src/main/scala/org/apache/zeppelin/rinterpreter/RContext.scala +++ b/r/src/main/scala/org/apache/zeppelin/rinterpreter/RContext.scala @@ -28,7 +28,7 @@ import org.apache.zeppelin.interpreter._ import org.apache.zeppelin.rinterpreter.rscala.RClient._ import org.apache.zeppelin.rinterpreter.rscala._ import org.apache.zeppelin.scheduler._ -import org.apache.zeppelin.spark.{SparkInterpreter, ZeppelinContext} +import org.apache.zeppelin.spark.{SparkInterpreter, SparkZeppelinContext} import org.slf4j._ import scala.collection.JavaConversions._ @@ -45,7 +45,7 @@ private[rinterpreter] class RContext(private val sockets: ScalaSockets, val backend: RBackendHelper = RBackendHelper() private var sc: Option[SparkContext] = None private
zeppelin git commit: ZEPPELIN-2151. Add integration test for livy cancel api
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 b9bc2b810 -> 59c1e1b0a ZEPPELIN-2151. Add integration test for livy cancel api ### What is this PR for? Just add integration test for livy cancel api. Only do it for livy 0.3 as the cancel api is only available in livy 0.3 ### What type of PR is it? [ Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2151 ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2056 from zjffdu/ZEPPELIN-2151 and squashes the following commits: b7ca7b3 [Jeff Zhang] ZEPPELIN-2151. Add integration test for livy cancel api (cherry picked from commit 152147122b9797baef20a382eb880eadcf7cdc0f) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/59c1e1b0 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/59c1e1b0 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/59c1e1b0 Branch: refs/heads/branch-0.7 Commit: 59c1e1b0abc68ac978fc2c4a0284e852a93f4fc7 Parents: b9bc2b8 Author: Jeff Zhang Authored: Wed Feb 22 19:59:47 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:48:51 2017 +0800 -- .../zeppelin/livy/BaseLivyInterprereter.java| 2 +- .../apache/zeppelin/livy/LivyInterpreterIT.java | 85 ++-- 2 files changed, 80 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/59c1e1b0/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index 7f92127..ecb5d77 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -55,7 +55,7 @@ public abstract class BaseLivyInterprereter extends Interpreter { private int pullStatusInterval; protected boolean displayAppInfo; private AtomicBoolean sessionExpired = new AtomicBoolean(false); - private LivyVersion livyVersion; + protected LivyVersion livyVersion; // keep tracking the mapping between paragraphId and statementId, so that we can cancel the // statement after we execute it. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/59c1e1b0/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java -- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index aec2742..7df9b20 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -82,13 +82,13 @@ public class LivyInterpreterIT { } InterpreterGroup interpreterGroup = new InterpreterGroup("group_1"); interpreterGroup.put("session_1", new ArrayList()); -LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties); +final LivySparkInterpreter sparkInterpreter = new LivySparkInterpreter(properties); sparkInterpreter.setInterpreterGroup(interpreterGroup); interpreterGroup.get("session_1").add(sparkInterpreter); AuthenticationInfo authInfo = new AuthenticationInfo("user1"); MyInterpreterOutputListener outputListener = new MyInterpreterOutputListener(); InterpreterOutput output = new InterpreterOutput(outputListener); -InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", +final InterpreterContext context = new InterpreterContext("noteId", "paragraphId", "livy.spark", "title", "text", authInfo, null, null, null, null, null, output); sparkInterpreter.open(); @@ -158,6 +158,31 @@ public class LivyInterpreterIT { assertEquals(InterpreterResult.Code.ERROR, result.code()); assertEquals(InterpreterResult.Type.TEXT, result.message().get(0).getType()); assertTrue(result.message().get(0).getData().contains("incomplete statement")); + + // cancel + if (sparkInterpreter.livyVersion.newerThanEquals(LivyVersion.LIVY_0_3_0)) { +Thread cancelThread = new Thread() { + @Override + public void run() { +// invoke cancel after 3 seconds to wait job starting +try { + Thread.sleep(3000); +
zeppelin git commit: ZEPPELIN-2261. Support to connect with livy through https
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 59c1e1b0a -> c52c0aced ZEPPELIN-2261. Support to connect with livy through https ### What is this PR for? Livy server support https, but the currently livy interpreter doesn't support it. This PR is for for the supporting to connect with livy through https ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2261 ### How should this be tested? Tested manually on livy server with ssl enabled. ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2139 from zjffdu/ZEPPELIN-2261 and squashes the following commits: 52fc204 [Jeff Zhang] address comment 53230c3 [Jeff Zhang] [ZEPPELIN-2261]. Support to connect with livy through https (cherry picked from commit 9d40013a994ca65f99831a416fb235d15ee17fdb) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c52c0ace Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c52c0ace Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c52c0ace Branch: refs/heads/branch-0.7 Commit: c52c0aced7bd265b20de2071d40e62690dd9e9af Parents: 59c1e1b Author: Jeff Zhang Authored: Mon Mar 20 16:11:38 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:50:17 2017 +0800 -- docs/interpreter/livy.md| 10 +++ .../zeppelin/livy/BaseLivyInterprereter.java| 83 .../zeppelin/livy/LivySQLInterpreterTest.java | 1 + 3 files changed, 79 insertions(+), 15 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c52c0ace/docs/interpreter/livy.md -- diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 6f04244..ce1c34a 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -130,6 +130,16 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` Adding extra libraries to livy interpreter + +zeppelin.livy.ssl.trustStore + +client trustStore file. Used when livy ssl is enabled + + +zeppelin.livy.ssl.trustStorePassword + +password for trustStore file. Used when livy ssl is enabled + **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.** http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c52c0ace/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index ecb5d77..7f9897e 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -21,6 +21,10 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.client.HttpClient; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.client.HttpClients; import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,11 +32,16 @@ import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; +import javax.net.ssl.SSLContext; +import java.io.FileInputStream; +import java.io.IOException; +import java.security.KeyStore; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -56,6 +65,7 @@ public abstract class BaseLivyInterprereter extends Interpreter { protected boolean displayAppInfo; private AtomicBoolean sessionExpired = new AtomicBoolean(false); protected LivyVersion livyVersion; + private RestTemplate restTemplate; // keep tracking the mapping between paragraphId and statementId, so that we can cancel the // statement after we execute it. @@ -70,6 +80,7 @@ public a
zeppelin git commit: ZEPPELIN-2224. Throw more meaningful exception when kerberos is enabled in livy interpreter
e.execute(RestTemplate.java:580) at org.springframework.web.client.RestTemplate.exchange(RestTemplate.java:498) at org.apache.zeppelin.livy.BaseLivyInterprereter.callRestAPI(BaseLivyInterprereter.java:407) at org.apache.zeppelin.livy.BaseLivyInterprereter.createSession(BaseLivyInterprereter.java:193) at org.apache.zeppelin.livy.BaseLivyInterprereter.initLivySession(BaseLivyInterprereter.java:99) ``` ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2104 from zjffdu/ZEPPELIN-2224 and squashes the following commits: 67feb4c [Jeff Zhang] code cleanup 0221193 [Jeff Zhang] ZEPPELIN-2224. Throw more meaningful exception when kerberos is enabled in livy interpreter (cherry picked from commit d5b46987f0e513bbaa20c681be01df8a32ae2674) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/b7171d19 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/b7171d19 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/b7171d19 Branch: refs/heads/branch-0.7 Commit: b7171d195e54a3c3ab733f3b6910555e2c29a8d5 Parents: c52c0ac Author: Jeff Zhang Authored: Tue Mar 7 15:38:29 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:54:44 2017 +0800 -- .../java/org/apache/zeppelin/livy/BaseLivyInterprereter.java| 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/b7171d19/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index 7f9897e..c744a0b 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -25,6 +25,7 @@ import org.apache.http.client.HttpClient; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; import org.apache.http.impl.client.HttpClients; +import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -358,13 +359,11 @@ public abstract class BaseLivyInterprereter extends Interpreter { } if (displayAppInfo) { -//TODO(zjffdu), use multiple InterpreterResult to display appInfo InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); interpreterResult.add(InterpreterResult.Type.TEXT, result); String appInfoHtml = "Spark Application Id: " + sessionInfo.appId + "" + "Spark WebUI: " + sessionInfo.webUIAddress + ""; -LOGGER.info("appInfoHtml:" + appInfoHtml); interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml); return interpreterResult; } else { @@ -475,6 +474,8 @@ public abstract class BaseLivyInterprereter extends Interpreter { if (cause.getResponseBodyAsString().matches(SESSION_NOT_FOUND_PATTERN)) { throw new SessionNotFoundException(cause.getResponseBodyAsString()); } +throw new LivyException(cause.getResponseBodyAsString() + "\n" ++ ExceptionUtils.getFullStackTrace(ExceptionUtils.getRootCause(e))); } throw new LivyException(e); }
zeppelin git commit: ZEPPELIN-2407. Livy Interpreter always return plain text result
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 b7171d195 -> d45fdf921 ZEPPELIN-2407. Livy Interpreter always return plain text result ### What is this PR for? It happens when zeppelin.livy.displayAppInfo is true. Straightforward fix. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2407 ### How should this be tested? Test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2254 from zjffdu/ZEPPELIN-2407 and squashes the following commits: ba59a77 [Jeff Zhang] ZEPPELIN-2407. Livy Interpreter always return plain text result (cherry picked from commit 5fd4ffc56658884b8f671ca40b1f78b8354f1f69) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d45fdf92 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d45fdf92 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d45fdf92 Branch: refs/heads/branch-0.7 Commit: d45fdf9216c5ef27d9cb48783062a2b326abeb39 Parents: b7171d1 Author: Jeff Zhang Authored: Sun Apr 16 16:38:36 2017 +0800 Committer: Jeff Zhang Committed: Fri Apr 28 08:56:13 2017 +0800 -- .../apache/zeppelin/livy/BaseLivyInterprereter.java | 2 +- .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 16 +++- 2 files changed, 16 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d45fdf92/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java index c744a0b..dfec98a 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterprereter.java @@ -360,7 +360,7 @@ public abstract class BaseLivyInterprereter extends Interpreter { if (displayAppInfo) { InterpreterResult interpreterResult = new InterpreterResult(InterpreterResult.Code.SUCCESS); -interpreterResult.add(InterpreterResult.Type.TEXT, result); +interpreterResult.add(result); String appInfoHtml = "Spark Application Id: " + sessionInfo.appId + "" + "Spark WebUI: " + sessionInfo.webUIAddress + ""; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d45fdf92/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java -- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 7df9b20..0ac76e6 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -147,6 +147,13 @@ public class LivyInterpreterIT { assertTrue(result.message().get(0).getData().contains("defined object Person")); } + // html output + String htmlCode = "println(\"%html hello \")"; + result = sparkInterpreter.interpret(htmlCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(1, result.message().size()); + assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + // error result = sparkInterpreter.interpret("println(a)", context); assertEquals(InterpreterResult.Code.ERROR, result.code()); @@ -425,8 +432,15 @@ public class LivyInterpreterIT { InterpreterResult result = sparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); assertEquals(2, result.message().size()); - assertTrue(result.message().get(1).getData().contains("Spark Application Id")); + + // html output + String htmlCode = "println(\"%html hello \")"; + result = sparkInterpreter.interpret(htmlCode, context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertEquals(2, result.message().size()); + assertEquals(InterpreterResult.Type.HTML, result.message().get(0).getType()); + } finally { sparkInterpreter.close(); }
zeppelin git commit: [ZEPPELIN-2465] Minor code fixes for the livy package
new LivyVersion((LivyVersionResponse.fromJson(callRestAPI("/version", "GET")).version)); - } - - private boolean isSessionExpired() throws LivyException { -try { - getSessionInfo(sessionInfo.id); - return false; -} catch (SessionNotFoundException e) { - return true; -} catch (LivyException e) { - throw e; -} - } - - private InterpreterResult appendSessionExpire(InterpreterResult result, boolean sessionExpired) { -if (sessionExpired) { - InterpreterResult result2 = new InterpreterResult(result.code()); - result2.add(InterpreterResult.Type.HTML, - "Previous livy session is expired, new livy session is created. " + - "Paragraphs that depend on this paragraph need to be re-executed!" + ""); - for (InterpreterResultMessage message : result.message()) { -result2.add(message.getType(), message.getData()); - } - return result2; -} else { - return result; -} - } - - - private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, - boolean displayAppInfo) { -if (stmtInfo.output != null && stmtInfo.output.isError()) { - return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue); -} else if (stmtInfo.isCancelled()) { - // corner case, output might be null if it is cancelled. - return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled"); -} else if (stmtInfo.output == null) { - // This case should never happen, just in case - return new InterpreterResult(InterpreterResult.Code.ERROR, "Empty output"); -} else { - //TODO(zjffdu) support other types of data (like json, image and etc) - String result = stmtInfo.output.data.plain_text; - - // check table magic result first - if (stmtInfo.output.data.application_livy_table_json != null) { -StringBuilder outputBuilder = new StringBuilder(); -boolean notFirstColumn = false; - -for (Map header : stmtInfo.output.data.application_livy_table_json.headers) { - if (notFirstColumn) { -outputBuilder.append("\t"); - } - outputBuilder.append(header.get("name")); - notFirstColumn = true; -} - -outputBuilder.append("\n"); -for (List row : stmtInfo.output.data.application_livy_table_json.records) { - outputBuilder.append(StringUtils.join(row, "\t")); - outputBuilder.append("\n"); -} -return new InterpreterResult(InterpreterResult.Code.SUCCESS, -InterpreterResult.Type.TABLE, outputBuilder.toString()); - } else if (stmtInfo.output.data.image_png != null) { -return new InterpreterResult(InterpreterResult.Code.SUCCESS, -InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png); - } else if (result != null) { -result = result.trim(); -if (result.startsWith("" -+ "Spark WebUI: " -+ sessionInfo.webUIAddress + ""; -interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml); -return interpreterResult; - } else { -return new InterpreterResult(InterpreterResult.Code.SUCCESS, result); - } -} - } - - private StatementInfo executeStatement(ExecuteRequest executeRequest) - throws LivyException { -return StatementInfo.fromJson(callRestAPI("/sessions/" + sessionInfo.id + "/statements", "POST", -executeRequest.toJson())); - } - - private StatementInfo getStatementInfo(int statementId) - throws LivyException { -return StatementInfo.fromJson( -callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId, "GET")); - } - - private void cancelStatement(int statementId) throws LivyException { -callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId + "/cancel", "POST"); - } - - - private RestTemplate createRestTemplate() { -HttpClient httpClient = null; -if (livyURL.startsWith("https:")) { - String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); - String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword"); - if (StringUtils.isBlank(keystoreFile)) { -throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl"); - } - if (StringUtils.isBlank(password)) { -throw new RuntimeException("No zeppelin.livy.ssl.trustStorePassword specified " + -"for livy ssl"); - } - FileInputStream inputStream =
zeppelin git commit: [ZEPPELIN-2465] Minor code fixes for the livy package
aragraphs that depend on this paragraph need to be re-executed!" + ""); - for (InterpreterResultMessage message : result.message()) { -result2.add(message.getType(), message.getData()); - } - return result2; -} else { - return result; -} - } - - - private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, - boolean displayAppInfo) { -if (stmtInfo.output != null && stmtInfo.output.isError()) { - return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue); -} else if (stmtInfo.isCancelled()) { - // corner case, output might be null if it is cancelled. - return new InterpreterResult(InterpreterResult.Code.ERROR, "Job is cancelled"); -} else if (stmtInfo.output == null) { - // This case should never happen, just in case - return new InterpreterResult(InterpreterResult.Code.ERROR, "Empty output"); -} else { - //TODO(zjffdu) support other types of data (like json, image and etc) - String result = stmtInfo.output.data.plain_text; - - // check table magic result first - if (stmtInfo.output.data.application_livy_table_json != null) { -StringBuilder outputBuilder = new StringBuilder(); -boolean notFirstColumn = false; - -for (Map header : stmtInfo.output.data.application_livy_table_json.headers) { - if (notFirstColumn) { -outputBuilder.append("\t"); - } - outputBuilder.append(header.get("name")); - notFirstColumn = true; -} - -outputBuilder.append("\n"); -for (List row : stmtInfo.output.data.application_livy_table_json.records) { - outputBuilder.append(StringUtils.join(row, "\t")); - outputBuilder.append("\n"); -} -return new InterpreterResult(InterpreterResult.Code.SUCCESS, -InterpreterResult.Type.TABLE, outputBuilder.toString()); - } else if (stmtInfo.output.data.image_png != null) { -return new InterpreterResult(InterpreterResult.Code.SUCCESS, -InterpreterResult.Type.IMG, (String) stmtInfo.output.data.image_png); - } else if (result != null) { -result = result.trim(); -if (result.startsWith("" -+ "Spark WebUI: " -+ sessionInfo.webUIAddress + ""; -interpreterResult.add(InterpreterResult.Type.HTML, appInfoHtml); -return interpreterResult; - } else { -return new InterpreterResult(InterpreterResult.Code.SUCCESS, result); - } -} - } - - private StatementInfo executeStatement(ExecuteRequest executeRequest) - throws LivyException { -return StatementInfo.fromJson(callRestAPI("/sessions/" + sessionInfo.id + "/statements", "POST", -executeRequest.toJson())); - } - - private StatementInfo getStatementInfo(int statementId) - throws LivyException { -return StatementInfo.fromJson( -callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId, "GET")); - } - - private void cancelStatement(int statementId) throws LivyException { -callRestAPI("/sessions/" + sessionInfo.id + "/statements/" + statementId + "/cancel", "POST"); - } - - - private RestTemplate createRestTemplate() { -HttpClient httpClient = null; -if (livyURL.startsWith("https:")) { - String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); - String password = property.getProperty("zeppelin.livy.ssl.trustStorePassword"); - if (StringUtils.isBlank(keystoreFile)) { -throw new RuntimeException("No zeppelin.livy.ssl.trustStore specified for livy ssl"); - } - if (StringUtils.isBlank(password)) { -throw new RuntimeException("No zeppelin.livy.ssl.trustStorePassword specified " + -"for livy ssl"); - } - FileInputStream inputStream = null; - try { -inputStream = new FileInputStream(keystoreFile); -KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); -trustStore.load(new FileInputStream(keystoreFile), password.toCharArray()); -SSLContext sslContext = SSLContexts.custom() -.loadTrustMaterial(trustStore) -.build(); -SSLConnectionSocketFactory csf = new SSLConnectionSocketFactory(sslContext); -httpClient = HttpClients.custom().setSSLSocketFactory(csf).build(); - } catch (Exception e) { -throw new RuntimeException("Failed to create SSL HttpClient", e); - } finally { -if (inputStream != null) { - try { -inputStream.close(); - } catch (IOException e) { -
zeppelin git commit: [HOTFIX]. Minor doc fix for back-end-angular.md
Repository: zeppelin Updated Branches: refs/heads/master 6c4ccf331 -> 2a06292c1 [HOTFIX]. Minor doc fix for back-end-angular.md ### What is this PR for? The example in the doc would fail due to typo, straightforward fix. ### What type of PR is it? [Hot Fix] ### Todos * [ ] - Task ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2307 from zjffdu/doc_hotfix and squashes the following commits: 133fe72 [Jeff Zhang] [HOTFIX]. Minor doc fix for back-end-angular.md Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/2a06292c Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/2a06292c Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/2a06292c Branch: refs/heads/master Commit: 2a06292c1fcc64ea4e503a29212a179f93699b07 Parents: 6c4ccf3 Author: Jeff Zhang Authored: Tue May 2 18:03:11 2017 +0800 Committer: Jeff Zhang Committed: Wed May 3 10:28:08 2017 +0800 -- docs/displaysystem/back-end-angular.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a06292c/docs/displaysystem/back-end-angular.md -- diff --git a/docs/displaysystem/back-end-angular.md b/docs/displaysystem/back-end-angular.md index 432036c..fb43ea4 100644 --- a/docs/displaysystem/back-end-angular.md +++ b/docs/displaysystem/back-end-angular.md @@ -112,7 +112,7 @@ import AngularElem._ ### Display Element ```scala // automatically convert to string and print with %angular display system directive in front. -.display +.display ``` ### Event Handler ```scala
zeppelin git commit: [HOTFIX]. Minor doc fix for back-end-angular.md
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 b9d8323b7 -> 4803f8ffb [HOTFIX]. Minor doc fix for back-end-angular.md ### What is this PR for? The example in the doc would fail due to typo, straightforward fix. ### What type of PR is it? [Hot Fix] ### Todos * [ ] - Task ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2307 from zjffdu/doc_hotfix and squashes the following commits: 133fe72 [Jeff Zhang] [HOTFIX]. Minor doc fix for back-end-angular.md (cherry picked from commit 2a06292c1fcc64ea4e503a29212a179f93699b07) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4803f8ff Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4803f8ff Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4803f8ff Branch: refs/heads/branch-0.7 Commit: 4803f8ffb7c065decceecc4c12f0334fdb37a15a Parents: b9d8323 Author: Jeff Zhang Authored: Tue May 2 18:03:11 2017 +0800 Committer: Jeff Zhang Committed: Wed May 3 10:28:23 2017 +0800 -- docs/displaysystem/back-end-angular.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4803f8ff/docs/displaysystem/back-end-angular.md -- diff --git a/docs/displaysystem/back-end-angular.md b/docs/displaysystem/back-end-angular.md index 432036c..fb43ea4 100644 --- a/docs/displaysystem/back-end-angular.md +++ b/docs/displaysystem/back-end-angular.md @@ -112,7 +112,7 @@ import AngularElem._ ### Display Element ```scala // automatically convert to string and print with %angular display system directive in front. -.display +.display ``` ### Event Handler ```scala
zeppelin git commit: ZEPPELIN-2486. AngularElem's onChange is only invoked once
Repository: zeppelin Updated Branches: refs/heads/master 08a981403 -> 92e3abc9d ZEPPELIN-2486. AngularElem's onChange is only invoked once ### What is this PR for? `ng-change` can not capture event, so that means `$event.timeStamp` is undefined. https://github.com/angular/angular.js/issues/6370 This cause AngularElem's onChange is only invoked once. This PR use another approach to update the angularObject. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2486 ### How should this be tested? Tested manually ### Screenshots (if appropriate) Before ![zeppelin_before](https://cloud.githubusercontent.com/assets/164491/25662178/227ec2ec-3046-11e7-9852-9e041d008698.gif) After ![zeppelin_after](https://cloud.githubusercontent.com/assets/164491/25662177/2108e898-3046-11e7-816f-7685480df83e.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2312 from zjffdu/ZEPPELIN-2486 and squashes the following commits: 4039d6b [Jeff Zhang] ZEPPELIN-2486. AngularElem's onChange is only invoked once Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/92e3abc9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/92e3abc9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/92e3abc9 Branch: refs/heads/master Commit: 92e3abc9d4a6a95da8440c6a4985234190741aae Parents: 08a9814 Author: Jeff Zhang Authored: Wed May 3 20:42:34 2017 +0800 Committer: Jeff Zhang Committed: Thu May 4 12:02:47 2017 +0800 -- .../apache/zeppelin/display/angular/AbstractAngularElem.scala| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/92e3abc9/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala -- diff --git a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala index cb6a567..7049e7a 100644 --- a/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala +++ b/zeppelin-display/src/main/scala/org/apache/zeppelin/display/angular/AbstractAngularElem.scala @@ -131,10 +131,10 @@ abstract class AbstractAngularElem(val interpreterContext: InterpreterContext, // create AngularFunction in current paragraph val functionName = eventName.replaceAll("-", "_") + "_" + uniqueId val elem = this % Attribute(None, eventName, - Text(s"${functionName}=$$event.timeStamp"), + Text(s"${functionName}=${functionName} + 1"), Null) -val angularObject = addAngularObject(functionName, "") +val angularObject = addAngularObject(functionName, 0) angularObject.addWatcher(new AngularObjectWatcher(interpreterContext) { override def watch(oldObject: scala.Any, newObject: scala.Any, context: InterpreterContext)
zeppelin git commit: [HOTFIX] Set interpreter context in zeppelin context from SparkSqlInterpreter
Repository: zeppelin Updated Branches: refs/heads/master 28ef8aa90 -> e5e3a88cd [HOTFIX] Set interpreter context in zeppelin context from SparkSqlInterpreter ### What is this PR for? When running `%sql` paragraphs sometimes a NullPointer is thrown instead of presenting the result. This was traced back to `sparkInterpreter.getZeppelinContext().setInterpreterContext(context);` not being called as expected from SparkSqlInterpreter. Since the SparkInterpreter is used from the SparkSqlInterpreter, if a `%spark` paragraph is executed before a `%sql` paragraph it works as expected (as SparkInterpreter sets the interpreter context in the zeppelin context). ### What type of PR is it? [Hot Fix] ### Todos ### What is the Jira issue? Hotfix ### How should this be tested? Start the spark context by executing ``` %sql select 1 ``` Expected result is a table with 1 in. Before this patch a NullPointer will be thrown. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: David Ersson Closes #2334 from FireArrow/sqlinterpreter_hotfix and squashes the following commits: 3d2b0a9 [David Ersson] Set interpreter context in zeppelin context Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e5e3a88c Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e5e3a88c Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e5e3a88c Branch: refs/heads/master Commit: e5e3a88cd434880af6319392a61ef0571f8319d6 Parents: 28ef8aa Author: David Ersson Authored: Wed May 10 18:03:19 2017 +0200 Committer: Jeff Zhang Committed: Thu May 11 10:27:43 2017 -0700 -- .../java/org/apache/zeppelin/spark/SparkSqlInterpreter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e5e3a88c/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java index 61c697c..d9e7563 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkSqlInterpreter.java @@ -96,7 +96,8 @@ public class SparkSqlInterpreter extends Interpreter { } sparkInterpreter.populateSparkWebUrl(context); -sqlc = getSparkInterpreter().getSQLContext(); +sparkInterpreter.getZeppelinContext().setInterpreterContext(context); +sqlc = sparkInterpreter.getSQLContext(); SparkContext sc = sqlc.sparkContext(); if (concurrentSQL()) { sc.setLocalProperty("spark.scheduler.pool", "fair"); @@ -126,7 +127,7 @@ public class SparkSqlInterpreter extends Interpreter { throw new InterpreterException(e); } -String msg = getSparkInterpreter().getZeppelinContext().showData(rdd); +String msg = sparkInterpreter.getZeppelinContext().showData(rdd); sc.clearJobGroup(); return new InterpreterResult(Code.SUCCESS, msg); }
zeppelin git commit: [ZEPPELIN-2564] - LivySparkSQLInterpreter throws NullPointerException when getProgress is called
Repository: zeppelin Updated Branches: refs/heads/master 2f03469bc -> 05af3bf4b [ZEPPELIN-2564] - LivySparkSQLInterpreter throws NullPointerException when getProgress is called ### What is this PR for? When using %livy.sql interpreter following error is logged into interpreter log: `ERROR [2017-05-17 15:01:32,897] ( {pool-1-thread-1} TThreadPoolServer.java[run]:296) - Error occurred during processing of message. java.lang.NullPointerException at org.apache.zeppelin.livy.BaseLivyInterpreter.getProgress(BaseLivyInterpreter.java:178) at org.apache.zeppelin.interpreter.LazyOpenInterpreter.getProgress(LazyOpenInterpreter.java:121) at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer.getProgress(RemoteInterpreterServer.java:558) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Processor$getProgress.getResult(RemoteInterpreterService.java:1899) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Processor$getProgress.getResult(RemoteInterpreterService.java:1884) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)` and zeppelin server log: `ERROR [2017-05-17 15:01:44,004] ( {JobProgressPoller, jobId=20170511-200435_1144682471} JobProgressPoller.java[run]:58) - Can not get or update progress org.apache.zeppelin.interpreter.InterpreterException: org.apache.thrift.transport.TTransportException at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getProgress(RemoteInterpreter.java:468) at org.apache.zeppelin.interpreter.LazyOpenInterpreter.getProgress(LazyOpenInterpreter.java:121) at org.apache.zeppelin.notebook.Paragraph.progress(Paragraph.java:334) at org.apache.zeppelin.scheduler.JobProgressPoller.run(JobProgressPoller.java:55) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Client.recv_getProgress(RemoteInterpreterService.java:321) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Client.getProgress(RemoteInterpreterService.java:306) at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getProgress(RemoteInterpreter.java:465) ... 3 more` This happens because this.livyVersion is never assigned in LivySparkSQLInterpreter: https://github.com/apache/zeppelin/blob/master/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java#L63 ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2564 ### How should this be tested? New test added to verify that livyVersion is not null ### Screenshots (if appropriate) ![errors](https://cloud.githubusercontent.com/assets/14924427/26164543/9aaa33a6-3b35-11e7-809d-e976277dac44.gif) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: sergeymazin Closes #2350 from sergeymazin/master and squashes the following commits: 3b95f5f [sergeymazin] Merge branch 'master' of https://github.com/sergeymazin/zeppelin b506753 [sergeymazin] Check sparkInterpreter for null and move test 630520a [sergeymazin] Check sparkInterpreter for null and move test c29c984 [sergeymazin] LivySparkSQLInterpreter to use getProgress of sparkInterpter 84db2d2 [sergeymazin] LivySparkSQLInterpreter throws NullPointerException when getProgress is called Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/05af3bf4 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/05af3bf4 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/05af3bf4 Branch: refs/heads/master Commit: 05af3bf4b7f68c8434a8d1cb576fb6f4571551ab Parents: 2f03469 Author: sergeymazin Authored: Thu May 18 01:13:44 2017 +0300 Committer: Jeff Zhang Committed: Thu May 18 10:34:08 2017 -0400 -- .../org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 9 + .../java/org/apache/zeppelin/livy/LivyInterpreterIT.java| 2 ++ 2 files changed, 11 insertions(+) --
zeppelin git commit: [ZEPPELIN-2564] - LivySparkSQLInterpreter throws NullPointerException when getProgress is called
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 5aff67be6 -> 9cccd8677 [ZEPPELIN-2564] - LivySparkSQLInterpreter throws NullPointerException when getProgress is called ### What is this PR for? When using %livy.sql interpreter following error is logged into interpreter log: `ERROR [2017-05-17 15:01:32,897] ( {pool-1-thread-1} TThreadPoolServer.java[run]:296) - Error occurred during processing of message. java.lang.NullPointerException at org.apache.zeppelin.livy.BaseLivyInterpreter.getProgress(BaseLivyInterpreter.java:178) at org.apache.zeppelin.interpreter.LazyOpenInterpreter.getProgress(LazyOpenInterpreter.java:121) at org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer.getProgress(RemoteInterpreterServer.java:558) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Processor$getProgress.getResult(RemoteInterpreterService.java:1899) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Processor$getProgress.getResult(RemoteInterpreterService.java:1884) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:285) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748)` and zeppelin server log: `ERROR [2017-05-17 15:01:44,004] ( {JobProgressPoller, jobId=20170511-200435_1144682471} JobProgressPoller.java[run]:58) - Can not get or update progress org.apache.zeppelin.interpreter.InterpreterException: org.apache.thrift.transport.TTransportException at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getProgress(RemoteInterpreter.java:468) at org.apache.zeppelin.interpreter.LazyOpenInterpreter.getProgress(LazyOpenInterpreter.java:121) at org.apache.zeppelin.notebook.Paragraph.progress(Paragraph.java:334) at org.apache.zeppelin.scheduler.JobProgressPoller.run(JobProgressPoller.java:55) Caused by: org.apache.thrift.transport.TTransportException at org.apache.thrift.transport.TIOStreamTransport.read(TIOStreamTransport.java:132) at org.apache.thrift.transport.TTransport.readAll(TTransport.java:86) at org.apache.thrift.protocol.TBinaryProtocol.readAll(TBinaryProtocol.java:429) at org.apache.thrift.protocol.TBinaryProtocol.readI32(TBinaryProtocol.java:318) at org.apache.thrift.protocol.TBinaryProtocol.readMessageBegin(TBinaryProtocol.java:219) at org.apache.thrift.TServiceClient.receiveBase(TServiceClient.java:69) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Client.recv_getProgress(RemoteInterpreterService.java:321) at org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService$Client.getProgress(RemoteInterpreterService.java:306) at org.apache.zeppelin.interpreter.remote.RemoteInterpreter.getProgress(RemoteInterpreter.java:465) ... 3 more` This happens because this.livyVersion is never assigned in LivySparkSQLInterpreter: https://github.com/apache/zeppelin/blob/master/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java#L63 ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2564 ### How should this be tested? New test added to verify that livyVersion is not null ### Screenshots (if appropriate) ![errors](https://cloud.githubusercontent.com/assets/14924427/26164543/9aaa33a6-3b35-11e7-809d-e976277dac44.gif) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: sergeymazin Closes #2350 from sergeymazin/master and squashes the following commits: 3b95f5f [sergeymazin] Merge branch 'master' of https://github.com/sergeymazin/zeppelin b506753 [sergeymazin] Check sparkInterpreter for null and move test 630520a [sergeymazin] Check sparkInterpreter for null and move test c29c984 [sergeymazin] LivySparkSQLInterpreter to use getProgress of sparkInterpter 84db2d2 [sergeymazin] LivySparkSQLInterpreter throws NullPointerException when getProgress is called (cherry picked from commit 05af3bf4b7f68c8434a8d1cb576fb6f4571551ab) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9cccd867 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9cccd867 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9cccd867 Branch: refs/heads/branch-0.7 Commit: 9cccd8677642bb52a29fa4a4d6e5d8ad8eb697b2 Parents: 5aff67b Author: sergeymazin Authored: Thu May 18 01:13:44 2017 +0300 Committer: Jeff Zhang Committed: Thu May 18 10:34:24 2017 -0400 -- .../org/apache/zeppelin/livy/LivySparkSQLInterpreter.java | 9 + .../java/org/apache
zeppelin git commit: [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl
Repository: zeppelin Updated Branches: refs/heads/master 88c5c3ccd -> e3ba23a9a [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl ### What is this PR for? We found that Livy doesn't work under kerberos and ssl, this is due to a bug when constructing KerberosRestTemplate. This PR fix this issue. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2584 ### How should this be tested? Unfornatelly it is difficult to add system test, so I manually tested it in a secure cluster. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: root Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits: 18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e3ba23a9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e3ba23a9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e3ba23a9 Branch: refs/heads/master Commit: e3ba23a9aa4909c7a8d3231e4d363771e0245b53 Parents: 88c5c3c Author: root Authored: Fri May 26 08:44:09 2017 + Committer: Jeff Zhang Committed: Thu Jun 1 15:17:21 2017 +0800 -- livy/pom.xml| 18 --- .../zeppelin/livy/BaseLivyInterpreter.java | 56 +--- pom.xml | 4 +- 3 files changed, 52 insertions(+), 26 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/pom.xml -- diff --git a/livy/pom.xml b/livy/pom.xml index 3c121fc..5f9dec7 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -37,14 +37,10 @@ 1.3 -4.3.4 4.3.0.RELEASE 1.0.1.RELEASE -3.2.4-Zeppelin -1.7.0 -1.9.5 0.3.0 2.1.0 2.6.0 @@ -80,7 +76,6 @@ org.apache.httpcomponents httpclient - ${httpcomponents.client.version} @@ -107,19 +102,6 @@ - org.assertj - assertj-core - ${assertj.version} - test - - - org.mockito - mockito-all - ${mockito.version} - test - - - com.cloudera.livy livy-integration-test ${livy.version} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e3ba23a9/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index b52ba16..fcb7e9d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -21,9 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; @@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.security.Principal; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -52,7 +63,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; - /** * Base class for livy interpreters. */ @@ -407,6 +417,11 @@ public abstract class BaseLivyInterpreter extends Interpreter { pr
zeppelin git commit: [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 a386253b7 -> 6a6fcc2ed [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl We found that Livy doesn't work under kerberos and ssl, this is due to a bug when constructing KerberosRestTemplate. This PR fix this issue. [Bug Fix] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-2584 Unfornatelly it is difficult to add system test, so I manually tested it in a secure cluster. * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: root Closes #2367 from zjffdu/ZEPPELIN-2584 and squashes the following commits: 18b23dc [root] [ZEPPELIN-2584]. Livy doesn't work under kerberos and ssl (cherry picked from commit e3ba23a9aa4909c7a8d3231e4d363771e0245b53) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/6a6fcc2e Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/6a6fcc2e Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/6a6fcc2e Branch: refs/heads/branch-0.7 Commit: 6a6fcc2ed6b0146e0a176232bedc0694fea43027 Parents: a386253 Author: root Authored: Fri May 26 08:44:09 2017 + Committer: Jeff Zhang Committed: Thu Jun 1 15:47:05 2017 +0800 -- livy/pom.xml| 4 -- .../zeppelin/livy/BaseLivyInterpreter.java | 55 ++-- pom.xml | 4 +- 3 files changed, 52 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/pom.xml -- diff --git a/livy/pom.xml b/livy/pom.xml index 5f41308..8a3d88d 100644 --- a/livy/pom.xml +++ b/livy/pom.xml @@ -37,14 +37,10 @@ 1.3 -4.3.4 4.3.0.RELEASE 1.0.1.RELEASE -3.2.4-Zeppelin -1.7.0 -1.9.5 0.3.0 2.1.0 2.6.0 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/6a6fcc2e/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index dd27e7d..1322a4d 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -21,9 +21,20 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.annotations.SerializedName; import org.apache.commons.lang.StringUtils; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.Credentials; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.impl.auth.SPNegoSchemeFactory; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.impl.client.HttpClients; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.zeppelin.interpreter.*; @@ -38,11 +49,11 @@ import org.springframework.security.kerberos.client.KerberosRestTemplate; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; - import javax.net.ssl.SSLContext; import java.io.FileInputStream; import java.io.IOException; import java.security.KeyStore; +import java.security.Principal; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -388,6 +399,11 @@ public abstract class BaseLivyInterpreter extends Interpreter { private RestTemplate createRestTemplate() { +String keytabLocation = property.getProperty("zeppelin.livy.keytab"); +String principal = property.getProperty("zeppelin.livy.principal"); +boolean isSpnegoEnabled = StringUtils.isNotEmpty(keytabLocation) && +StringUtils.isNotEmpty(principal); + HttpClient httpClient = null; if (livyURL.startsWith("https:")) { String keystoreFile = property.getProperty("zeppelin.livy.ssl.trustStore"); @@ -408,7 +424,37 @@ public abstract class BaseLivyInterpreter extends Interpreter { .loadTrustMaterial(trustStore)
zeppelin git commit: [ZEPPELIN-2589]Let zeppelin show more than one line of error log returned by livy.
Repository: zeppelin Updated Branches: refs/heads/master 43926485c -> 380cfbfe3 [ZEPPELIN-2589]Let zeppelin show more than one line of error log returned by livy. ### What is this PR for? Let zeppelin show more than one line of error log returned by livy. ### What type of PR is it? Improvement ### What is the Jira issue? [ZEPPELIN-2589 ](https://issues.apache.org/jira/browse/ZEPPELIN-2589) ### How should this be tested? 1. build: mvn clean package -DskipTests; 2. /bin/zeppelin-daemon.sh restart 3. Start a notebook 4. Use Livy interpreter and input some erroneous spark code, check the shown result is more than one line ### Screenshots (if appropriate) When using Livy 0.4 to run spark streaming job. Original Zeppelin: ![image](https://user-images.githubusercontent.com/14201792/26881018-c9aaf9bc-4bc8-11e7-802a-b655656c1baf.png) After add traceback to interpreter result: ![image](https://user-images.githubusercontent.com/14201792/26881987-a7de9d68-4bcb-11e7-8e20-e77a5e1f7033.png) ### Questions: * Does the licenses files need update?No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: æ±ªèµ«æ¬ Closes #2390 from qwemicheal/master and squashes the following commits: ee3b42c [汪赫æ¬] ci fail randomly 6e4d2aa [汪赫æ¬] trigger ci c7e9b0e [汪赫æ¬] minor change to trigger ci 0b657a8 [汪赫æ¬] add check for traceback and modify way of checking msg number 4ea3edb [汪赫æ¬] merger evalue and traceback to one message and use join to concatenate traceback b1caa2c [汪赫æ¬] minor change e729db2 [汪赫æ¬] merge unit test to pyspark test case 774b370 [汪赫æ¬] use stringBuilder to concat traceback lines f8de47a [汪赫æ¬] code style fixed f96abbb [汪赫æ¬] only use pyspark de2e06b [汪赫æ¬] close the pyspark interpreter after test 0212e35 [汪赫æ¬] change to pyspark for livy 0.2 when testing traceback msg 799f1c2 [汪赫æ¬] take livy livy version into consideration e3f9791 [汪赫æ¬] add unit test bf26dbe [汪赫æ¬] whitespace fitting f6fc200 [汪赫æ¬] fix traceback type mismatch 3310055 [汪赫æ¬] add try catch to find which string failed ci e306d4f [汪赫æ¬] add white space after comma 3307c6f [汪赫æ¬] add more debug msg 6a2983f [汪赫æ¬] fix typo 04061d3 [汪赫æ¬] add debug print 5a03e05 [汪赫æ¬] add check for livy 0.2 when traceback field is missing 45c5d9f [汪赫æ¬] add traceback msg from livy return json to InterpreterResult f6135cc [汪赫æ¬] add traceback msg from livy return json to InterpreterResult 2eb6011 [汪赫æ¬] add traceback msg from livy return json to InterpreterResult 905c54d [汪赫æ¬] add traceback msg from livy return json to InterpreterResult Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/380cfbfe Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/380cfbfe Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/380cfbfe Branch: refs/heads/master Commit: 380cfbfe3cdef7e9281ad39b0b7e2cd3b161f72b Parents: 4392648 Author: æ±ªèµ«æ¬ Authored: Wed Jun 7 21:31:40 2017 +0800 Committer: Jeff Zhang Committed: Sat Jun 10 09:46:54 2017 +0800 -- .../zeppelin/livy/BaseLivyInterpreter.java | 27 +--- .../apache/zeppelin/livy/LivyInterpreterIT.java | 21 ++- 2 files changed, 44 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/380cfbfe/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index fcb7e9d..523dfad 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -341,7 +341,17 @@ public abstract class BaseLivyInterpreter extends Interpreter { private InterpreterResult getResultFromStatementInfo(StatementInfo stmtInfo, boolean displayAppInfo) { if (stmtInfo.output != null && stmtInfo.output.isError()) { - return new InterpreterResult(InterpreterResult.Code.ERROR, stmtInfo.output.evalue); + InterpreterResult result = new InterpreterResult(InterpreterResult.Code.ERROR); + StringBuilder sb = new StringBuilder(); + sb.append(stmtInfo.output.evalue); + // in case evalue doesn't have newline char + if (!stmtInfo.output.evalue.contains("\n")) +sb.append("\n"); + if (stmtInfo.output.traceback != null) { +sb.append(StringUtils.join(stmtInfo.output.traceback)); + } + result.add(sb.toString()); + return result; } else if (stmtIn
zeppelin git commit: [ZEPPELIN-2688]. Upgrade ace to 1.2.7
Repository: zeppelin Updated Branches: refs/heads/master 4eea33ad5 -> 19eb80a24 [ZEPPELIN-2688]. Upgrade ace to 1.2.7 ### What is this PR for? Ace 1.2.7 add syntax highlight for pig script. This ticket is to upgrade ace from 1.2.6 to 1.2.7 ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2688 ### How should this be tested? Tested manually ### Screenshots (if appropriate) Before ![before](https://user-images.githubusercontent.com/164491/27514859-802a8a24-59c8-11e7-9631-e19fcbb2f11e.png) After ![after](https://user-images.githubusercontent.com/164491/27514861-81a22bfa-59c8-11e7-8b88-239d42a1588e.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2435 from zjffdu/ZEPPELIN-2688 and squashes the following commits: 99b2554 [Jeff Zhang] [ZEPPELIN-2688]. Upgrade ace to 1.2.7 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/19eb80a2 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/19eb80a2 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/19eb80a2 Branch: refs/heads/master Commit: 19eb80a243b350952fefd3fb2f644e3ca1d12446 Parents: 4eea33a Author: Jeff Zhang Authored: Sun Jun 25 17:03:45 2017 +0800 Committer: Jeff Zhang Committed: Thu Jun 29 14:17:03 2017 +0800 -- zeppelin-web/bower.json | 7 --- zeppelin-web/karma.conf.js | 1 + zeppelin-web/src/index.html | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/19eb80a2/zeppelin-web/bower.json -- diff --git a/zeppelin-web/bower.json b/zeppelin-web/bower.json index 8e9a1e5..a68c2e9 100644 --- a/zeppelin-web/bower.json +++ b/zeppelin-web/bower.json @@ -14,7 +14,7 @@ "angular-resource": "1.5.7", "angular-bootstrap": "~2.5.0", "angular-websocket": "~1.0.13", -"ace-builds": "1.2.6", +"ace-builds": "1.2.7", "angular-ui-ace": "0.1.3", "jquery.scrollTo": "~1.4.13", "nvd3": "~1.8.5", @@ -46,13 +46,14 @@ "src-noconflict/mode-python.js", "src-noconflict/mode-sql.js", "src-noconflict/mode-markdown.js", +"src-noconflict/mode-pig.js", "src-noconflict/mode-sh.js", "src-noconflict/mode-r.js", "src-noconflict/keybinding-emacs.js", "src-noconflict/ext-language_tools.js", "src-noconflict/theme-chrome.js" ], - "version": "1.2.6", + "version": "1.2.7", "name": "ace-builds" }, "highlightjs": { @@ -65,7 +66,7 @@ } }, "resolutions": { -"ace-builds": "1.2.6", +"ace-builds": "1.2.7", "angular": ">=1.5.0 <1.6" } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/19eb80a2/zeppelin-web/karma.conf.js -- diff --git a/zeppelin-web/karma.conf.js b/zeppelin-web/karma.conf.js index b163b56..58cf6bc 100644 --- a/zeppelin-web/karma.conf.js +++ b/zeppelin-web/karma.conf.js @@ -59,6 +59,7 @@ module.exports = function(config) { 'bower_components/ace-builds/src-noconflict/mode-python.js', 'bower_components/ace-builds/src-noconflict/mode-sql.js', 'bower_components/ace-builds/src-noconflict/mode-markdown.js', + 'bower_components/ace-builds/src-noconflict/mode-pig.js', 'bower_components/ace-builds/src-noconflict/mode-sh.js', 'bower_components/ace-builds/src-noconflict/mode-r.js', 'bower_components/ace-builds/src-noconflict/keybinding-emacs.js', http://git-wip-us.apache.org/repos/asf/zeppelin/blob/19eb80a2/zeppelin-web/src/index.html -- diff --git a/zeppelin-web/src/index.html b/zeppelin-web/src/index.html index 7d48584..547fd3a 100644 --- a/zeppelin-web/src/index.html +++ b/zeppelin-web/src/index.html @@ -140,6 +140,7 @@ limitations under the License. +
zeppelin git commit: ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine
Repository: zeppelin Updated Branches: refs/heads/master 19eb80a24 -> f04ddfbe7 ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine ### What is this PR for? Pig 0.17.0 has just been released. This PR is to upgrade pig to 0.17.0 and support spark engine which is a big milestone of pig 0.17.0 Main Changes: * Upgrade pig to 0.17.0 * Remove some code using java reflection in `PigUtils.java`, as pig 0.17.0 has some improvement and expose new apis which could be used pig interpreter. * Support spark engine ### What type of PR is it? [Improvement | Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://github.com/zjffdu/zeppelin/tree/ZEPPELIN-2615 ### How should this be tested? Unit test is added and also manually test spark yarn-client mode in pig tutorial note. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2431 from zjffdu/ZEPPELIN-2615 and squashes the following commits: d4e9a6d [Jeff Zhang] Address comments 4b4e3db [Jeff Zhang] ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/f04ddfbe Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/f04ddfbe Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/f04ddfbe Branch: refs/heads/master Commit: f04ddfbe71f95ef02938f9508ee84ed01f1992a7 Parents: 19eb80a Author: Jeff Zhang Authored: Sun Jun 25 10:08:12 2017 +0800 Committer: Jeff Zhang Committed: Thu Jun 29 14:20:23 2017 +0800 -- LICENSE | 7 + docs/interpreter/pig.md | 41 +++- pig/pom.xml | 47 +++- .../org/apache/zeppelin/pig/PigInterpreter.java | 12 +- .../zeppelin/pig/PigQueryInterpreter.java | 11 +- .../java/org/apache/zeppelin/pig/PigUtils.java | 241 +-- pig/src/main/resources/interpreter-setting.json | 20 +- .../zeppelin/pig/PigInterpreterSparkTest.java | 149 8 files changed, 267 insertions(+), 261 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f04ddfbe/LICENSE -- diff --git a/LICENSE b/LICENSE index e206a6c..a6c02de 100644 --- a/LICENSE +++ b/LICENSE @@ -276,3 +276,10 @@ The following components are provided under the BSD 2-Clause license. See file (BSD 2 Clause) portions of SQLLine (http://sqlline.sourceforge.net/) - http://sqlline.sourceforge.net/#license jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java + + +Jython Software License + +The following components are provided under the Jython Software License. See file headers and project links for details. + + (Jython Software License) jython-standalone - http://www.jython.org/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f04ddfbe/docs/interpreter/pig.md -- diff --git a/docs/interpreter/pig.md b/docs/interpreter/pig.md index 2e633a2..128ea76 100644 --- a/docs/interpreter/pig.md +++ b/docs/interpreter/pig.md @@ -29,31 +29,39 @@ which in turns enables them to handle very large data sets. - No pig alias in the last statement in `%pig.query` (read the examples below). - The last statement must be in single line in `%pig.query` -## Supported runtime mode - - Local - - MapReduce - - Tez_Local (Only Tez 0.7 is supported) - - Tez (Only Tez 0.7 is supported) ## How to use -### How to setup Pig +### How to setup Pig execution modes. - Local Mode -Nothing needs to be done for local mode +Set `zeppelin.pig.execType` as `local`. - MapReduce Mode -HADOOP\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. +Set `zeppelin.pig.execType` as `mapreduce`. HADOOP\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. - Tez Local Mode -Nothing needs to be done for tez local mode +Only Tez 0.7 is supported. Set `zeppelin.pig.execType` as `tez_local`. - Tez Mode -HADOOP\_CONF\_DIR and TEZ\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. +Only Tez 0.7 is supported. Set `zeppelin.pig.execType` as `tez`. HADOOP\_CONF\_DIR and TEZ\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. + +- Spark Local Mode + +Only Spark 1.6.x is supported, by default it is Spark 1.6.3. Set `zeppelin.pig.execType`
zeppelin git commit: ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 ff1a35306 -> c3c6c7a45 ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine ### What is this PR for? Pig 0.17.0 has just been released. This PR is to upgrade pig to 0.17.0 and support spark engine which is a big milestone of pig 0.17.0 Main Changes: * Upgrade pig to 0.17.0 * Remove some code using java reflection in `PigUtils.java`, as pig 0.17.0 has some improvement and expose new apis which could be used pig interpreter. * Support spark engine ### What type of PR is it? [Improvement | Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://github.com/zjffdu/zeppelin/tree/ZEPPELIN-2615 ### How should this be tested? Unit test is added and also manually test spark yarn-client mode in pig tutorial note. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2431 from zjffdu/ZEPPELIN-2615 and squashes the following commits: d4e9a6d [Jeff Zhang] Address comments 4b4e3db [Jeff Zhang] ZEPPELIN-2615. Upgrade pig to 0.17.0 to support spark engine (cherry picked from commit f04ddfbe71f95ef02938f9508ee84ed01f1992a7) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c3c6c7a4 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c3c6c7a4 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c3c6c7a4 Branch: refs/heads/branch-0.7 Commit: c3c6c7a450f5d24e1a89ba482ad88ee454af4a61 Parents: ff1a353 Author: Jeff Zhang Authored: Sun Jun 25 10:08:12 2017 +0800 Committer: Jeff Zhang Committed: Thu Jun 29 14:20:44 2017 +0800 -- LICENSE | 7 + docs/interpreter/pig.md | 41 +++- pig/pom.xml | 47 +++- .../org/apache/zeppelin/pig/PigInterpreter.java | 12 +- .../zeppelin/pig/PigQueryInterpreter.java | 11 +- .../java/org/apache/zeppelin/pig/PigUtils.java | 241 +-- pig/src/main/resources/interpreter-setting.json | 20 +- .../zeppelin/pig/PigInterpreterSparkTest.java | 149 8 files changed, 267 insertions(+), 261 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c3c6c7a4/LICENSE -- diff --git a/LICENSE b/LICENSE index b8b84ef..e36d231 100644 --- a/LICENSE +++ b/LICENSE @@ -273,3 +273,10 @@ The following components are provided under the BSD 2-Clause license. See file (BSD 2 Clause) portions of SQLLine (http://sqlline.sourceforge.net/) - http://sqlline.sourceforge.net/#license jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java + + +Jython Software License + +The following components are provided under the Jython Software License. See file headers and project links for details. + + (Jython Software License) jython-standalone - http://www.jython.org/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c3c6c7a4/docs/interpreter/pig.md -- diff --git a/docs/interpreter/pig.md b/docs/interpreter/pig.md index d1f18fa..4dd840b 100644 --- a/docs/interpreter/pig.md +++ b/docs/interpreter/pig.md @@ -25,31 +25,39 @@ group: manual - No pig alias in the last statement in `%pig.query` (read the examples below). - The last statement must be in single line in `%pig.query` -## Supported runtime mode - - Local - - MapReduce - - Tez_Local (Only Tez 0.7 is supported) - - Tez (Only Tez 0.7 is supported) ## How to use -### How to setup Pig +### How to setup Pig execution modes. - Local Mode -Nothing needs to be done for local mode +Set `zeppelin.pig.execType` as `local`. - MapReduce Mode -HADOOP\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. +Set `zeppelin.pig.execType` as `mapreduce`. HADOOP\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. - Tez Local Mode -Nothing needs to be done for tez local mode +Only Tez 0.7 is supported. Set `zeppelin.pig.execType` as `tez_local`. - Tez Mode -HADOOP\_CONF\_DIR and TEZ\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. +Only Tez 0.7 is supported. Set `zeppelin.pig.execType` as `tez`. HADOOP\_CONF\_DIR and TEZ\_CONF\_DIR needs to be specified in `ZEPPELIN_HOME/conf/zeppelin-env.sh`. + +- Spark Local Mode + +Only Spark 1.6.x is supported, by defa
[2/2] zeppelin git commit: ZEPPELIN-2397. Add toJson and fromJson method for serializable objects
ZEPPELIN-2397. Add toJson and fromJson method for serializable objects ### What is this PR for? Add toJson and fromJson method for all the objects that need ser/der in zeppelin. ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2397?filter=-2 ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2272 from zjffdu/ZEPPELIN-2397 and squashes the following commits: 4fb12fe [Jeff Zhang] ZEPPELIN-2397. Add toJson and fromJson method for serializable objects Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/f36b1a15 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/f36b1a15 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/f36b1a15 Branch: refs/heads/master Commit: f36b1a1577ccd7dce438cf834985c9d72abca1e3 Parents: f3f24f3 Author: Jeff Zhang Authored: Wed Apr 12 18:43:34 2017 +0800 Committer: Jeff Zhang Committed: Tue Jul 4 18:24:16 2017 +0800 -- .../zeppelin/common/JsonSerializable.java | 26 +++ .../org/apache/zeppelin/dep/Repository.java | 14 +++- .../apache/zeppelin/display/AngularObject.java | 15 +++- .../java/org/apache/zeppelin/display/GUI.java | 35 - .../java/org/apache/zeppelin/display/Input.java | 67 .../apache/zeppelin/helium/HeliumPackage.java | 15 +++- .../zeppelin/interpreter/InterpreterResult.java | 14 +++- .../RemoteZeppelinServerResource.java | 15 +++- .../InvokeResourceMethodEventMessage.java | 14 +++- .../remote/RemoteInterpreterEventClient.java| 19 ++--- .../remote/RemoteInterpreterEventPoller.java| 19 +++-- .../remote/RemoteInterpreterServer.java | 17 ++-- .../zeppelin/resource/RemoteResource.java | 15 +++- .../org/apache/zeppelin/resource/Resource.java | 14 +++- .../apache/zeppelin/resource/ResourceId.java| 15 +++- .../zeppelin/resource/ResourcePoolUtils.java| 7 +- .../apache/zeppelin/resource/ResourceSet.java | 15 +++- .../zeppelin/user/AuthenticationInfo.java | 16 +++- .../org/apache/zeppelin/user/Credentials.java | 3 +- .../zeppelin/user/CredentialsInfoSaving.java| 15 +++- .../org/apache/zeppelin/display/GUITest.java| 49 .../zeppelin/helium/HeliumPackageTest.java | 4 +- .../apache/zeppelin/realm/ZeppelinHubRealm.java | 20 +++-- .../org/apache/zeppelin/rest/HeliumRestApi.java | 2 +- .../zeppelin/rest/InterpreterRestApi.java | 12 +-- .../zeppelin/rest/NotebookRepoRestApi.java | 7 +- .../apache/zeppelin/rest/NotebookRestApi.java | 10 +-- .../zeppelin/rest/message/CronRequest.java | 14 +++- .../message/NewInterpreterSettingRequest.java | 13 +++- .../zeppelin/rest/message/NewNoteRequest.java | 14 +++- .../rest/message/NewParagraphRequest.java | 15 +++- .../message/NotebookRepoSettingsRequest.java| 13 +++- .../rest/message/RestartInterpreterRequest.java | 15 +++- .../RunParagraphWithParametersRequest.java | 15 +++- .../UpdateInterpreterSettingRequest.java| 14 +++- .../apache/zeppelin/socket/NotebookServer.java | 4 +- .../zeppelin/rest/AbstractTestRestApi.java | 2 - .../zeppelin/rest/ZeppelinSparkClusterTest.java | 3 - .../zeppelin/socket/NotebookServerTest.java | 12 +-- .../java/org/apache/zeppelin/helium/Helium.java | 13 +--- .../helium/HeliumApplicationFactory.java| 4 +- .../zeppelin/helium/HeliumBundleFactory.java| 9 +-- .../org/apache/zeppelin/helium/HeliumConf.java | 19 - .../org/apache/zeppelin/helium/NpmPackage.java | 15 +++- .../apache/zeppelin/helium/WebpackResult.java | 15 +++- .../interpreter/InterpreterFactory.java | 72 - .../interpreter/InterpreterInfoSaving.java | 16 +++- .../interpreter/InterpreterSettingManager.java | 8 +- .../interpreter/remote/RemoteInterpreter.java | 4 +- .../java/org/apache/zeppelin/notebook/Note.java | 80 --- .../org/apache/zeppelin/notebook/Notebook.java | 14 +--- .../notebook/NotebookAuthorization.java | 3 +- .../NotebookAuthorizationInfoSaving.java| 16 +++- .../notebook/NotebookImportDeserializer.java| 3 +- .../org/apache/zeppelin/notebook/Paragraph.java | 71 - .../json/NotebookTypeAdapterFactory.java| 82 .../notebook/repo/AzureNotebookRepo.java| 15 +--- .../notebook/repo/MongoNotebookRepo.java| 11 +-- .../zeppelin/notebook/repo/S3NotebookRepo.java | 14 +--- .../zeppelin/notebook/repo/VFSNotebookRepo.java | 11 +-- .../zeppelinhub/websocket/ZeppelinClient.java | 20
[1/2] zeppelin git commit: ZEPPELIN-2397. Add toJson and fromJson method for serializable objects
import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -35,6 +39,7 @@ import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; +import org.apache.zeppelin.notebook.json.NotebookTypeAdapterFactory; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.utility.IdHashes; import org.apache.zeppelin.resource.ResourcePoolUtils; @@ -52,12 +57,26 @@ import com.google.gson.Gson; /** * Binded interpreters for a note */ -public class Note implements Serializable, ParagraphJobListener { +public class Note implements ParagraphJobListener, JsonSerializable { private static final Logger logger = LoggerFactory.getLogger(Note.class); private static final long serialVersionUID = 7920699076577612429L; - private static final Gson gson = new GsonBuilder() - .registerTypeAdapterFactory(Input.TypeAdapterFactory) - .create(); + private static Gson gson = new GsonBuilder() + .setPrettyPrinting() + .registerTypeAdapterFactory(new NotebookTypeAdapterFactory(Paragraph.class) { +@Override +protected void beforeWrite(Paragraph source, JsonElement toSerialize) { + Map runtimeInfos = source.getRuntimeInfos(); + if (runtimeInfos != null) { +JsonElement jsonTree = gson.toJsonTree(runtimeInfos); +if (toSerialize instanceof JsonObject) { + JsonObject jsonObj = (JsonObject) toSerialize; + jsonObj.add("runtimeInfos", jsonTree); +} + } +} + }).setDateFormat("-MM-dd HH:mm:ss.SSS") + .registerTypeAdapter(Date.class, new NotebookImportDeserializer()) + .registerTypeAdapterFactory(Input.TypeAdapterFactory).create(); // threadpool for delayed persist of note private static final ScheduledThreadPoolExecutor delayedPersistThreadPool = @@ -323,7 +342,7 @@ public class Note implements Serializable, ParagraphJobListener { try { Gson gson = new Gson(); String resultJson = gson.toJson(srcParagraph.getReturn()); - InterpreterResult result = gson.fromJson(resultJson, InterpreterResult.class); + InterpreterResult result = InterpreterResult.fromJson(resultJson); newParagraph.setReturn(result, null); } catch (Exception e) { // 'result' part of Note consists of exception, instead of actual interpreter results @@ -898,11 +917,6 @@ public class Note implements Serializable, ParagraphJobListener { } public static Note fromJson(String json) { -GsonBuilder gsonBuilder = -new GsonBuilder(); -gsonBuilder.setPrettyPrinting(); -Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()) -.create(); Note note = gson.fromJson(json, Note.class); convertOldInput(note); return note; @@ -913,4 +927,50 @@ public class Note implements Serializable, ParagraphJobListener { p.settings.convertOldInput(); } } + + @Override + public boolean equals(Object o) { +if (this == o) { + return true; +} +if (o == null || getClass() != o.getClass()) { + return false; +} +Note note = (Note) o; + +if (paragraphs != null ? !paragraphs.equals(note.paragraphs) : note.paragraphs != null) { + return false; +} +//TODO(zjffdu) exclude name because FolderView.index use Note as key and consider different name +//as same note +//if (name != null ? !name.equals(note.name) : note.name != null) return false; +if (id != null ? !id.equals(note.id) : note.id != null) { + return false; +} +if (angularObjects != null ? +!angularObjects.equals(note.angularObjects) : note.angularObjects != null) { + return false; +} +if (config != null ? !config.equals(note.config) : note.config != null) { + return false; +} +return info != null ? info.equals(note.info) : note.info == null; + + } + + @Override + public int hashCode() { +int result = paragraphs != null ? paragraphs.hashCode() : 0; +//result = 31 * result + (name != null ? name.hashCode() : 0); +result = 31 * result + (id != null ? id.hashCode() : 0); +result = 31 * result + (angularObjects != null ? angularObjects.hashCode() : 0); +result = 31 * result + (config != null ? config.hashCode() : 0); +result = 31 * result + (info != null ? info.hashCode() : 0); +return result; + } + + @VisibleForTesting + public static Gson getGson() { +return gson; + } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f36b1a15/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java -- diff --git
zeppelin git commit: [ZEPPELIN-2716] Change the default value of zeppelin.livy.displayAppInfo to true
Repository: zeppelin Updated Branches: refs/heads/master ad3138385 -> 985b86e4d [ZEPPELIN-2716] Change the default value of zeppelin.livy.displayAppInfo to true ### What is this PR for? Since it is quite useful to expose the application info for user to monitor and debug, so here propose to enable "zeppelin.livy.displayAppInfo" to true as default. ### What type of PR is it? Improvement ### Todos ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2716 ### How should this be tested? Manual verification. Author: jerryshao Closes #2459 from jerryshao/ZEPPELIN-2716 and squashes the following commits: c9506ab [jerryshao] Address UT failure 32d3f72 [jerryshao] Change the default value of zeppelin.livy.displayAppInfo to true Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/985b86e4 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/985b86e4 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/985b86e4 Branch: refs/heads/master Commit: 985b86e4d2754d3c73422e5c722c8a5e62bc1b40 Parents: ad31383 Author: jerryshao Authored: Wed Jul 5 22:27:02 2017 +0800 Committer: Jeff Zhang Committed: Tue Jul 11 16:39:30 2017 +0800 -- docs/interpreter/livy.md| 16 .../apache/zeppelin/livy/BaseLivyInterpreter.java | 2 +- livy/src/main/resources/interpreter-setting.json| 6 +++--- .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 1 + 4 files changed, 13 insertions(+), 12 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/985b86e4/docs/interpreter/livy.md -- diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index d85b02a..1741a80 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -43,7 +43,7 @@ We added some common configurations for spark, and you can set any configuration You can find all Spark configurations in [here](http://spark.apache.org/docs/latest/configuration.html#available-properties). And instead of starting property with `spark.` it should be replaced with `livy.spark.`. Example: `spark.driver.memory` to `livy.spark.driver.memory` - + Property @@ -72,7 +72,7 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` zeppelin.livy.displayAppInfo -false +true Whether to display app info @@ -150,7 +150,7 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.** ## Adding External libraries -You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version. +You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version. Example @@ -166,7 +166,7 @@ Example Adding extra libraries to livy interpreter - + ## How to use Basically, you can use @@ -197,9 +197,9 @@ hello("livy") ``` ## Impersonation -When Zeppelin server is running with authentication enabled, -then this interpreter utilizes Livyâs user impersonation feature -i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). +When Zeppelin server is running with authentication enabled, +then this interpreter utilizes Livyâs user impersonation feature +i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server. ## Apply Zeppelin Dynamic Forms @@ -228,4 +228,4 @@ The session would have timed out, you may need to restart the interpreter. Edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line. If you choose to work on livy in `apps/spark/java` directory in [https://github.com/cloudera/hue](https://github.com/cloudera/hue), -copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`. +copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/985b86e4/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java --
zeppelin git commit: [ZEPPELIN-2716] Change the default value of zeppelin.livy.displayAppInfo to true
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 3355b8f25 -> 5636f9355 [ZEPPELIN-2716] Change the default value of zeppelin.livy.displayAppInfo to true Since it is quite useful to expose the application info for user to monitor and debug, so here propose to enable "zeppelin.livy.displayAppInfo" to true as default. Improvement https://issues.apache.org/jira/browse/ZEPPELIN-2716 Manual verification. Author: jerryshao Closes #2459 from jerryshao/ZEPPELIN-2716 and squashes the following commits: c9506ab [jerryshao] Address UT failure 32d3f72 [jerryshao] Change the default value of zeppelin.livy.displayAppInfo to true (cherry picked from commit 985b86e4d2754d3c73422e5c722c8a5e62bc1b40) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5636f935 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5636f935 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5636f935 Branch: refs/heads/branch-0.7 Commit: 5636f9355a80bdf10e5fb564b9db71dd06059039 Parents: 3355b8f Author: jerryshao Authored: Wed Jul 5 22:27:02 2017 +0800 Committer: Jeff Zhang Committed: Tue Jul 11 16:42:34 2017 +0800 -- docs/interpreter/livy.md| 16 ++-- .../apache/zeppelin/livy/BaseLivyInterpreter.java | 2 +- livy/src/main/resources/interpreter-setting.json| 9 + .../org/apache/zeppelin/livy/LivyInterpreterIT.java | 1 + 4 files changed, 17 insertions(+), 11 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5636f935/docs/interpreter/livy.md -- diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index ce1c34a..132f327 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -43,7 +43,7 @@ We added some common configurations for spark, and you can set any configuration You can find all Spark configurations in [here](http://spark.apache.org/docs/latest/configuration.html#available-properties). And instead of starting property with `spark.` it should be replaced with `livy.spark.`. Example: `spark.driver.memory` to `livy.spark.driver.memory` - + Property @@ -67,7 +67,7 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` zeppelin.livy.displayAppInfo -false +true Whether to display app info @@ -145,7 +145,7 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.** ## Adding External libraries -You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version. +You can load dynamic library to livy interpreter by set `livy.spark.jars.packages` property to comma-separated list of maven coordinates of jars to include on the driver and executor classpaths. The format for the coordinates should be groupId:artifactId:version. Example @@ -161,7 +161,7 @@ Example Adding extra libraries to livy interpreter - + ## How to use Basically, you can use @@ -192,7 +192,11 @@ hello("livy") ``` ## Impersonation -When Zeppelin server is running with authentication enabled, then this interpreter utilizes Livyâs user impersonation feature i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). This is particularly useful when multi users are sharing a Notebook server. + +When Zeppelin server is running with authentication enabled, +then this interpreter utilizes Livyâs user impersonation feature +i.e. sends extra parameter for creating and running a session ("proxyUser": "${loggedInUser}"). +This is particularly useful when multi users are sharing a Notebook server. ## Apply Zeppelin Dynamic Forms @@ -221,4 +225,4 @@ The session would have timed out, you may need to restart the interpreter. Edit `conf/spark-blacklist.conf` file in livy server and comment out `#spark.master` line. If you choose to work on livy in `apps/spark/java` directory in [https://github.com/cloudera/hue](https://github.com/cloudera/hue), -copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`. +copy `spark-user-configurable-options.template` to `spark-user-configurable-options.conf` file in livy server and comment out `#spark.master`. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5636f935/livy/src/ma
zeppelin git commit: [ZEPPELIN-2768]. Bump up Spark version to 2.2.0
Repository: zeppelin Updated Branches: refs/heads/master 3c737ef36 -> ebb6591d8 [ZEPPELIN-2768]. Bump up Spark version to 2.2.0 ### What is this PR for? Spark 2.2.0 is just released, this PR is to support spark 2.2.0. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2768 ### How should this be tested? Travis is updated ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2486 from zjffdu/ZEPPELIN-2768 and squashes the following commits: 5adc2e5 [Jeff Zhang] [ZEPPELIN-2768]. Bump up Spark version to 2.2.0 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/ebb6591d Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/ebb6591d Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/ebb6591d Branch: refs/heads/master Commit: ebb6591d84881a1c0408bbaf27073dedb387c411 Parents: 3c737ef Author: Jeff Zhang Authored: Wed Jul 12 11:51:47 2017 +0800 Committer: Jeff Zhang Committed: Fri Jul 14 15:09:40 2017 +0800 -- .travis.yml | 8 ++-- spark-dependencies/pom.xml | 12 +++- spark/pom.xml | 12 +++- .../java/org/apache/zeppelin/spark/SparkVersion.java| 4 ++-- 4 files changed, 30 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/ebb6591d/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 219d555..a883de1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,8 +65,8 @@ matrix: # Several tests were excluded from this configuration due to the following issues: # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470 # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property -- jdk: "oraclejdk7" - env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" +- jdk: "oraclejdk8" + env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" @@ -76,6 +76,10 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" +# Test spark module for 2.2.0 with scala 2.11, livy +- jdk: "oraclejdk8" + env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy"
zeppelin git commit: [ZEPPELIN-2768]. Bump up Spark version to 2.2.0
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 d59ff44b7 -> 28310c2b9 [ZEPPELIN-2768]. Bump up Spark version to 2.2.0 Spark 2.2.0 is just released, this PR is to support spark 2.2.0. [Improvement] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-2768 Travis is updated * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2486 from zjffdu/ZEPPELIN-2768 and squashes the following commits: 5adc2e5 [Jeff Zhang] [ZEPPELIN-2768]. Bump up Spark version to 2.2.0 (cherry picked from commit ebb6591d84881a1c0408bbaf27073dedb387c411) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/28310c2b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/28310c2b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/28310c2b Branch: refs/heads/branch-0.7 Commit: 28310c2b95785d8b9e63bc0adc5a26df8b3c9dec Parents: d59ff44 Author: Jeff Zhang Authored: Wed Jul 12 11:51:47 2017 +0800 Committer: Jeff Zhang Committed: Fri Jul 14 15:12:07 2017 +0800 -- .travis.yml | 4 spark-dependencies/pom.xml | 12 +++- spark/pom.xml | 12 +++- .../java/org/apache/zeppelin/spark/SparkVersion.java| 4 ++-- 4 files changed, 28 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/28310c2b/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index c7e97d8..10eba61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,6 +57,10 @@ matrix: - jdk: "oraclejdk7" env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" +# Test spark module for 2.2.0 with scala 2.11, livy +- jdk: "oraclejdk8" + env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" + # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.1 -Phadoop-2.6 -Ppyspark -Psparkr -Pscala-2.11" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/28310c2b/spark-dependencies/pom.xml -- diff --git a/spark-dependencies/pom.xml b/spark-dependencies/pom.xml index 74035a4..ffdc754 100644 --- a/spark-dependencies/pom.xml +++ b/spark-dependencies/pom.xml @@ -532,11 +532,21 @@ spark-2.1 + +2.1.0 +2.5.0 +0.10.4 +2.11.8 + + + + + spark-2.2 true -2.1.0 +2.2.0 2.5.0 0.10.4 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/28310c2b/spark/pom.xml -- diff --git a/spark/pom.xml b/spark/pom.xml index 9531880..3d005eb 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -525,11 +525,21 @@ spark-2.1 + +2.1.0 +2.5.0 +0.10.4 +2.11.8 + + + + + spark-2.2 true -2.1.0 +2.2.0 2.5.0 0.10.4 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/28310c2b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersi
zeppelin git commit: [ZEPPELIN-2758]. NumberFormatException on importing notebook
Repository: zeppelin Updated Branches: refs/heads/master 87705a932 -> 3dd25c24e [ZEPPELIN-2758]. NumberFormatException on importing notebook ### What is this PR for? This is due to the custom TypeAdapter for Paragraph. I don't know why we introduce it. To me it is not necessary. So I remove this in this PR. And the spark UI still can be accessed in frontend. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2758 ### How should this be tested? Unit test is updated and manually tested ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2485 from zjffdu/ZEPPELIN-2758 and squashes the following commits: 74f8488 [Jeff Zhang] reset runtimeInfos b51d566 [Jeff Zhang] [ZEPPELIN-2758]. NumberFormatException on importing notebook Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3dd25c24 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3dd25c24 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3dd25c24 Branch: refs/heads/master Commit: 3dd25c24e42538dfc0bfb2d7967c6a928624daea Parents: 87705a9 Author: Jeff Zhang Authored: Sat Jul 15 14:21:15 2017 +0800 Committer: Jeff Zhang Committed: Wed Jul 19 15:27:54 2017 +0800 -- .../json/NotebookTypeAdapterFactory.java| 85 .../apache/zeppelin/socket/NotebookServer.java | 15 +--- .../zeppelin/socket/NotebookServerTest.java | 3 +- .../java/org/apache/zeppelin/notebook/Note.java | 23 ++ .../org/apache/zeppelin/notebook/Paragraph.java | 6 ++ .../json/NotebookTypeAdapterFactory.java| 82 --- 6 files changed, 17 insertions(+), 197 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3dd25c24/zeppelin-server/src/main/java/org/apache/zeppelin/json/NotebookTypeAdapterFactory.java -- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/json/NotebookTypeAdapterFactory.java b/zeppelin-server/src/main/java/org/apache/zeppelin/json/NotebookTypeAdapterFactory.java deleted file mode 100644 index a22c03b..000 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/json/NotebookTypeAdapterFactory.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.json; - -import java.io.IOException; - -import org.apache.zeppelin.socket.NotebookServer; - -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.TypeAdapter; -import com.google.gson.TypeAdapterFactory; -import com.google.gson.reflect.TypeToken; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; - -/** - * Custom adapter type factory - * Modify the jsonObject before serailaization/deserialization - * Check sample implementation at {@link NotebookServer} - * @param the type whose json is to be customized for serialization/deserialization - */ -public class NotebookTypeAdapterFactory implements TypeAdapterFactory { - private final Class customizedClass; - - public NotebookTypeAdapterFactory(Class customizedClass) { -this.customizedClass = customizedClass; - } - - @SuppressWarnings("unchecked") - // we use a runtime check to guarantee that 'C' and 'T' are equal - public final TypeAdapter create(Gson gson, TypeToken type) { -return type.getRawType() == customizedClass ? (TypeAdapter) customizeTypeAdapter(gson, -(TypeToken) type) : null; - } - - private TypeAdapter customizeTypeAdapter(Gson gson, TypeToken type) { -final TypeAdapter delegate = gson.getDelegateAdapter(this, type); -final TypeAdapter elementAdapter = gson.getAdapter(JsonElement.class); -return new TypeAdapter() { - @Override - pub
zeppelin git commit: ZEPPELIN-1263. Should specify zeppelin's spark configuration through --conf arguments of spark-submit
Repository: zeppelin Updated Branches: refs/heads/master e1f20ddda -> 1b9c46dcf ZEPPELIN-1263. Should specify zeppelin's spark configuration through --conf arguments of spark-submit ### What is this PR for? For now we spark configuration at runtime rather than pass them through `--conf`, it would cause several issues. - Some configuration has to be set through --conf, otherwise we need to duplicate code in SparkSubmit.scala (spark.yarn.keytab, spark.yarn.principal) - Some configuration would conflict with spark-defaults.conf. If you specify spark.master as yarn-client in spark-defaults.conf but specify spark.master as local in zeppelin side, you will see the spark interpreter fail to start due to this inconsistency. - As ZEPPELIN-1460 described, it is hard to figure what is the effective configuration. - We can not use yarn-cluster mode although it is not supported now, but I think it is necessary to do that as zeppelin needs to support multiple users. This PR would pass all the spark related configuration to spark-submit through `--conf`, so that it is easy to know and guarantee that configuration on zeppelin interpreter setting take precedence over spark-defaults.conf. And it is also good for maintenance that upstream change (any change about configuration in spark) would not affect us. ### What type of PR is it? [Improvement] ### Todos - [ ] - Task ### What is the Jira issue? - https://issues.apache.org/jira/browse/ZEPPELIN-1263 ### How should this be tested? Tested spark 1.6 spark 2.0 on both yarn-client mode and embedded mode. ### Screenshots (if appropriate) ![image](https://cloud.githubusercontent.com/assets/164491/18702212/3e7b54d0-8013-11e6-95f7-502b3cf89d67.png) ### Questions: - Does the licenses files need update? No - Is there breaking changes for older versions? No - Does this needs documentation? No ⦠Author: Jeff Zhang Closes #1446 from zjffdu/ZEPPELIN-1263 and squashes the following commits: f57d7bb [Jeff Zhang] address comments 151e991 [Jeff Zhang] ZEPPELIN-1263. Should specify zeppelin's spark configuration through --conf arguments of spark-submit Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1b9c46dc Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1b9c46dc Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1b9c46dc Branch: refs/heads/master Commit: 1b9c46dcfd2bec6e0c1a2d4f1aa41d2834d23c77 Parents: e1f20dd Author: Jeff Zhang Authored: Mon Jul 17 13:26:15 2017 +0800 Committer: Jeff Zhang Committed: Sat Jul 22 20:30:44 2017 +0800 -- bin/interpreter.sh | 4 +-- .../apache/zeppelin/spark/SparkInterpreter.java | 25 +- .../interpreter/remote/RemoteInterpreter.java | 36 +++- 3 files changed, 38 insertions(+), 27 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1b9c46dc/bin/interpreter.sh -- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 29d0221..1344e31 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -202,9 +202,9 @@ fi if [[ -n "${SPARK_SUBMIT}" ]]; then if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}` else - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${PORT}` fi else INTERPRETER_RUN_COMMAND+=' '` echo ${ZEPPELIN_RUNNER} ${JAVA_INTP_OPTS} $
zeppelin git commit: [HOTFIX] Followup of ZEPPELIN-1263
Repository: zeppelin Updated Branches: refs/heads/master c5109ba8a -> f2c2941cf [HOTFIX] Followup of ZEPPELIN-1263 ### What is this PR for? It is a hotfix of ZEPPELIN-1263, will merge it when CI pass ### What type of PR is it? [ Hot Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1263 ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2501 from zjffdu/HOTFIX_1236 and squashes the following commits: b589c6b [Jeff Zhang] [HOTFIX] Followup of ZEPPELIN-1236 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/f2c2941c Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/f2c2941c Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/f2c2941c Branch: refs/heads/master Commit: f2c2941cf1dd2311a0a0b87bf5573a5d66b56d67 Parents: c5109ba Author: Jeff Zhang Authored: Mon Jul 24 16:26:50 2017 +0800 Committer: Jeff Zhang Committed: Mon Jul 24 19:55:20 2017 +0800 -- .../apache/zeppelin/interpreter/remote/RemoteInterpreter.java| 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f2c2941c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java index 847153e..12e0caa 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreter.java @@ -151,8 +151,8 @@ public class RemoteInterpreter extends Interpreter { sparkConfBuilder.append(" --master " + property.getProperty("master")); } if (isSparkConf(key, property.getProperty(key))) { -sparkConfBuilder.append(" --conf " + key + "=\"" + -toShellFormat(property.getProperty(key)) + "\""); +sparkConfBuilder.append(" --conf " + key + "=" + +toShellFormat(property.getProperty(key))); } } env.put("ZEPPELIN_SPARK_CONF", sparkConfBuilder.toString());
zeppelin git commit: ZEPPELIN-2811. Specify precise explicitly for travis build
Repository: zeppelin Updated Branches: refs/heads/master 90b3be5b6 -> 9367f87b3 ZEPPELIN-2811. Specify precise explicitly for travis build ### What is this PR for? Travis is upgrading ubuntu from precise to trusty, so sometimes your travis will fails if it is routed to trusty. I confirmed that with travis support ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2811 ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2504 from zjffdu/ZEPPELIN-2811 and squashes the following commits: 7e0aa0a [Jeff Zhang] ZEPPELIN-2811. Specify precise explicitly for travis build Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9367f87b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9367f87b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9367f87b Branch: refs/heads/master Commit: 9367f87b399f39be5e8ab0060125be14678f523f Parents: 90b3be5 Author: Jeff Zhang Authored: Wed Jul 26 11:39:27 2017 +0800 Committer: Jeff Zhang Committed: Wed Jul 26 17:37:16 2017 +0800 -- .travis.yml | 11 +++ 1 file changed, 11 insertions(+) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9367f87b/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index a883de1..099fb38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,6 +43,7 @@ matrix: include: # Test License compliance using RAT tool - jdk: "oraclejdk7" + dist: precise env: SCALA_VER="2.11" SPARK_VER="2.0.2" HADOOP_VER="2.6" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS="" # Run e2e tests (in zeppelin-web) @@ -66,43 +67,53 @@ matrix: # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470 # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property - jdk: "oraclejdk8" + dist: precise env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FLAG="package -Pbuild-distr -DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_PROJECTS="-Dtests.to.exclude=**/ZeppelinSparkClusterTest.java,**/org.apache.zeppelin.spark.*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false" # Test selenium with spark module for 1.6.3 - jdk: "oraclejdk7" + dist: precise env: TEST_SELENIUM="true" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-1.6 -Phadoop-2.6 -Phelium-dev -Pexamples" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" TEST_PROJECTS="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false" # Test interpreter modules - jdk: "oraclejdk7" + dist: precise env: SCALA_VER="2.10" PROFILE="-Pscalding" BUILD_FLAG="package -DskipTests -DskipRat -Pr" TEST_FLAG="test -DskipRat" MODULES="-pl $(echo .,zeppelin-interpreter,${INTERPRETERS} | sed 's/!//g')" TEST_PROJECTS="" # Test spark module for 2.2.0 with scala 2.11, livy - jdk: "oraclejdk8" + dist: precise env: SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.2 -Phadoop-2.6 -Pscala-2.11" SPARKR="true" BUILD_FLAG="package -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark,livy" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.livy.* -DfailIfNoTests=false" # Test spark module for 2.1.0 with scala 2.11, livy - jdk: "oraclejdk7" + dist: precise env: SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Pweb-ci -Pspark-2.1 -Phad
zeppelin git commit: ZEPPELIN-1515. Notebook: HDFS as a backend storage (Use hadoop client jar)
Repository: zeppelin Updated Branches: refs/heads/master dfc62f5f4 -> 30bfcae0c ZEPPELIN-1515. Notebook: HDFS as a backend storage (Use hadoop client jar) ### What is this PR for? This PR is trying to add hdfs as another implementation for `NotebookRepo`. There's another PR about using webhdfs to implement that. Actually hdfs client library is compatibility cross major versions. See http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/Compatibility.html#Wire_compatibility, if using webhdfs, the code become more complicated and may lose some features of hdfs. This PR is also required for HA of zeppelin, so that multiple zeppelin instances can share notes via hdfs. I add hadoop-client in pom file. So zeppelin will package hadoop client jar into its binary distribution. This is because zeppelin may be installed in a gateway machine where no hadoop is installed (only hadoop configuration file is existed in this machine) And since the hadoop client will work with multiple versions of hadoop, so it is fine to package into binary distribution. Spark also package hadoop client jar in its binary distribution. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1515 ### How should this be tested? Unit test is added. Also manually verify it in a single node cluster. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2455 from zjffdu/ZEPPELIN-1515 and squashes the following commits: b3e83ab [Jeff Zhang] ZEPPELIN-1515. Notebook: HDFS as a backend storage (Read & Write Mode) Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/30bfcae0 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/30bfcae0 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/30bfcae0 Branch: refs/heads/master Commit: 30bfcae0c0c9650aff3ed1f8fe41eee9c4e93cb1 Parents: dfc62f5 Author: Jeff Zhang Authored: Fri Jun 30 14:48:22 2017 +0800 Committer: Jeff Zhang Committed: Thu Aug 24 08:29:09 2017 +0800 -- bin/zeppelin-daemon.sh | 4 + bin/zeppelin.sh | 4 + conf/zeppelin-site.xml.template | 20 ++ docs/setup/storage/storage.md | 17 ++ zeppelin-server/pom.xml | 89 - zeppelin-zengine/pom.xml| 66 ++ .../zeppelin/conf/ZeppelinConfiguration.java| 5 +- .../java/org/apache/zeppelin/notebook/Note.java | 1 + .../notebook/repo/HdfsNotebookRepo.java | 200 +++ .../notebook/repo/NotebookRepoSync.java | 1 + .../notebook/repo/HdfsNotebookRepoTest.java | 101 ++ 11 files changed, 418 insertions(+), 90 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/30bfcae0/bin/zeppelin-daemon.sh -- diff --git a/bin/zeppelin-daemon.sh b/bin/zeppelin-daemon.sh index e88c26f..5982aee 100755 --- a/bin/zeppelin-daemon.sh +++ b/bin/zeppelin-daemon.sh @@ -67,6 +67,10 @@ if [[ -d "${ZEPPELIN_HOME}/zeppelin-server/target/classes" ]]; then ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-server/target/classes" fi +if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" +fi + # Add jdbc connector jar # ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/jdbc/jars/jdbc-connector-jar" http://git-wip-us.apache.org/repos/asf/zeppelin/blob/30bfcae0/bin/zeppelin.sh -- diff --git a/bin/zeppelin.sh b/bin/zeppelin.sh index 44fc2cf..a13f9db 100755 --- a/bin/zeppelin.sh +++ b/bin/zeppelin.sh @@ -73,6 +73,10 @@ addJarInDir "${ZEPPELIN_HOME}/zeppelin-web/target/lib" ZEPPELIN_CLASSPATH="$CLASSPATH:$ZEPPELIN_CLASSPATH" +if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then + ZEPPELIN_CLASSPATH+=":${HADOOP_CONF_DIR}" +fi + if [[ ! -d "${ZEPPELIN_LOG_DIR}" ]]; then echo "Log dir doesn't exist, create ${ZEPPELIN_LOG_DIR}" $(mkdir -p "${ZEPPELIN_LOG_DIR}") http://git-wip-us.apache.org/repos/asf/zeppelin/blob/30bfcae0/conf/zeppelin-site.xml.template -- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index adf5810..3ec6e27 100755 --- a/conf/zeppelin-site.xml.template +++ b/co
[2/3] zeppelin git commit: [ZEPPELIN-2753] Basic Implementation of IPython Interpreter
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/32517c9d/python/src/main/resources/grpc/python/ipython_pb2.py -- diff --git a/python/src/main/resources/grpc/python/ipython_pb2.py b/python/src/main/resources/grpc/python/ipython_pb2.py new file mode 100644 index 000..eca3dfe --- /dev/null +++ b/python/src/main/resources/grpc/python/ipython_pb2.py @@ -0,0 +1,751 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: ipython.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='ipython.proto', + package='ipython', + syntax='proto3', + serialized_pb=_b('\n\ripython.proto\x12\x07ipython\"\x1e\n\x0e\x45xecuteRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\"l\n\x0f\x45xecuteResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.ExecuteStatus\x12!\n\x04type\x18\x02 \x01(\x0e\x32\x13.ipython.OutputType\x12\x0e\n\x06output\x18\x03 \x01(\t\"\x0f\n\rCancelRequest\"\x10\n\x0e\x43\x61ncelResponse\"1\n\x11\x43ompletionRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0e\n\x06\x63ursor\x18\x02 \x01(\x05\"%\n\x12\x43ompletionResponse\x12\x0f\n\x07matches\x18\x01 \x03(\t\"\x0f\n\rStatusRequest\"8\n\x0eStatusResponse\x12&\n\x06status\x18\x01 \x01(\x0e\x32\x16.ipython.IPythonStatus\"\r\n\x0bStopRequest\"\x0e\n\x0cStopResponse*\'\n\rExecuteStatus\x12\x0b\n\x07SUCCESS\x10\x00\x12\t\n\x05\x45RROR\x10\x01**\n\rIPythonStatus\x12\x0c\n\x08STARTING\x10\x00\x12\x0b\n\x07RUNNING\x10\x01*!\n\nOutputType\x12\x08\n\x04TEXT\x10\x00\x12\t\n\x05IMAGE\x10\x01\x32\xc3\x02\n\x07IPython\x12@\n\x07\x65xecute\x12\x17.ipython.ExecuteRequest\x1a\x1 8.ipython.ExecuteResponse\"\x00\x30\x01\x12\x45\n\x08\x63omplete\x12\x1a.ipython.CompletionRequest\x1a\x1b.ipython.CompletionResponse\"\x00\x12;\n\x06\x63\x61ncel\x12\x16.ipython.CancelRequest\x1a\x17.ipython.CancelResponse\"\x00\x12;\n\x06status\x12\x16.ipython.StatusRequest\x1a\x17.ipython.StatusResponse\"\x00\x12\x35\n\x04stop\x12\x14.ipython.StopRequest\x1a\x15.ipython.StopResponse\"\x00\x42<\n org.apache.zeppelin.python.protoB\x0cIPythonProtoP\x01\xa2\x02\x07IPythonb\x06proto3') +) + +_EXECUTESTATUS = _descriptor.EnumDescriptor( + name='ExecuteStatus', + full_name='ipython.ExecuteStatus', + filename=None, + file=DESCRIPTOR, + values=[ +_descriptor.EnumValueDescriptor( + name='SUCCESS', index=0, number=0, + options=None, + type=None), +_descriptor.EnumValueDescriptor( + name='ERROR', index=1, number=1, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=399, + serialized_end=438, +) +_sym_db.RegisterEnumDescriptor(_EXECUTESTATUS) + +ExecuteStatus = enum_type_wrapper.EnumTypeWrapper(_EXECUTESTATUS) +_IPYTHONSTATUS = _descriptor.EnumDescriptor( + name='IPythonStatus', + full_name='ipython.IPythonStatus', + filename=None, + file=DESCRIPTOR, + values=[ +_descriptor.EnumValueDescriptor( + name='STARTING', index=0, number=0, + options=None, + type=None), +_descriptor.EnumValueDescriptor( + name='RUNNING', index=1, number=1, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=440, + serialized_end=482, +) +_sym_db.RegisterEnumDescriptor(_IPYTHONSTATUS) + +IPythonStatus = enum_type_wrapper.EnumTypeWrapper(_IPYTHONSTATUS) +_OUTPUTTYPE = _descriptor.EnumDescriptor( + name='OutputType', + full_name='ipython.OutputType', + filename=None, + file=DESCRIPTOR, + values=[ +_descriptor.EnumValueDescriptor( + name='TEXT', index=0, number=0, + options=None, + type=None), +_descriptor.EnumValueDescriptor( + name='IMAGE', index=
[3/3] zeppelin git commit: [ZEPPELIN-2753] Basic Implementation of IPython Interpreter
[ZEPPELIN-2753] Basic Implementation of IPython Interpreter ### What is this PR for? This is the first step for implement IPython Interpreter in Zeppelin. I just use the jupyter_client to create and manage the ipython kernel. We don't need to care about python compilation and execution, all the things are delegated to ipython kernel. Ideally all the features of ipython should be available in Zeppelin as well. For now, user can use %python.ipython for IPython Interpreter. And if ipython is available, the default python interpreter will use ipython. But user can still set `zeppelin.python.useIPython` as false to enforce to use the old implementation of python interpreter. Main features: * IPython interpreter support ** All the ipython features are available, including visualization, ipython magics. * ZeppelinContext support * Streaming output support * Support Ipython in PySpark Regarding the visualization, ideally all the visualization libraries work in jupyter should also work here. In unit test, I only verify the following 3 popular visualization library. could add more later. * matplotlib * bokeh * ggplot ### What type of PR is it? [Feature ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2753 ### How should this be tested? Unit test is added. ### Screenshots (if appropriate) Verify bokeh in IPython Interpreter ![image](https://user-images.githubusercontent.com/164491/27999716-756d749e-6552-11e7-90bb-4c6b08f4ab5c.png) Verify matplotlib ![image](https://user-images.githubusercontent.com/164491/28046960-e881b28e-6619-11e7-9e1f-7f4662f842f3.png) Verify ZeppelinContext ![image](https://user-images.githubusercontent.com/164491/28119378-4212620c-6747-11e7-89d5-3b5e609593ce.png) Verify Streaming ![streaming](https://user-images.githubusercontent.com/164491/28950974-8f92fe1e-78fa-11e7-841f-3174da198bb7.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2474 from zjffdu/ZEPPELIN-2753 and squashes the following commits: e869f31 [Jeff Zhang] address comments b0b5c95 [Jeff Zhang] [ZEPPELIN-2753] Basic Implementation of IPython Interpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/32517c9d Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/32517c9d Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/32517c9d Branch: refs/heads/master Commit: 32517c9d9fbdc2235560388a47f9e3eff4ec4854 Parents: 26a39df Author: Jeff Zhang Authored: Sat Aug 26 11:59:43 2017 +0800 Committer: Jeff Zhang Committed: Mon Aug 28 08:11:04 2017 +0800 -- .travis.yml | 24 +- alluxio/pom.xml | 1 + docs/interpreter/python.md | 64 ++ docs/interpreter/spark.md | 6 + pom.xml | 13 +- python/README.md| 19 + python/pom.xml | 65 ++ .../apache/zeppelin/python/IPythonClient.java | 211 ++ .../zeppelin/python/IPythonInterpreter.java | 359 + .../zeppelin/python/PythonInterpreter.java | 66 ++ .../zeppelin/python/PythonZeppelinContext.java | 49 ++ python/src/main/proto/ipython.proto | 102 +++ python/src/main/resources/grpc/generate_rpc.sh | 18 + .../resources/grpc/python/ipython_client.py | 36 + .../main/resources/grpc/python/ipython_pb2.py | 751 +++ .../resources/grpc/python/ipython_pb2_grpc.py | 129 .../resources/grpc/python/ipython_server.py | 155 .../resources/grpc/python/zeppelin_python.py| 107 +++ .../src/main/resources/interpreter-setting.json | 23 + .../zeppelin/python/IPythonInterpreterTest.java | 402 ++ .../python/PythonInterpreterMatplotlibTest.java | 1 + .../python/PythonInterpreterPandasSqlTest.java | 38 +- .../zeppelin/python/PythonInterpreterTest.java | 2 + python/src/test/resources/log4j.properties | 31 + spark/pom.xml | 51 +- .../zeppelin/spark/IPySparkInterpreter.java | 110 +++ .../zeppelin/spark/PySparkInterpreter.java | 92 ++- .../src/main/resources/interpreter-setting.json | 17 + .../main/resources/python/zeppelin_ipyspark.py | 53 ++ .../sparkr-resources/interpreter-setting.json | 11 + .../zeppelin/spark/IPySparkInterpreterTest.java | 203 + .../spark/PySparkInterpreterMatplotlibTest.java | 18 +- .../zeppelin/spark/PySparkInterpreterTest.java | 36 +- spark/src/test/resources/log4j.properties | 2 + testing/install_external_dependencies.sh| 3 +- zeppelin-interpreter/pom.xml| 5 - .../interpreter/BaseZeppelinContext
[1/3] zeppelin git commit: [ZEPPELIN-2753] Basic Implementation of IPython Interpreter
Repository: zeppelin Updated Branches: refs/heads/master 26a39df08 -> 32517c9d9 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/32517c9d/spark/src/main/resources/python/zeppelin_ipyspark.py -- diff --git a/spark/src/main/resources/python/zeppelin_ipyspark.py b/spark/src/main/resources/python/zeppelin_ipyspark.py new file mode 100644 index 000..324f481 --- /dev/null +++ b/spark/src/main/resources/python/zeppelin_ipyspark.py @@ -0,0 +1,53 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +from py4j.java_gateway import java_import, JavaGateway, GatewayClient +from pyspark.conf import SparkConf +from pyspark.context import SparkContext + +# for back compatibility +from pyspark.sql import SQLContext + +# start JVM gateway +client = GatewayClient(port=${JVM_GATEWAY_PORT}) +gateway = JavaGateway(client, auto_convert=True) + +java_import(gateway.jvm, "org.apache.spark.SparkEnv") +java_import(gateway.jvm, "org.apache.spark.SparkConf") +java_import(gateway.jvm, "org.apache.spark.api.java.*") +java_import(gateway.jvm, "org.apache.spark.api.python.*") +java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*") + +intp = gateway.entry_point +jsc = intp.getJavaSparkContext() + +java_import(gateway.jvm, "org.apache.spark.sql.*") +java_import(gateway.jvm, "org.apache.spark.sql.hive.*") +java_import(gateway.jvm, "scala.Tuple2") + +jconf = jsc.getConf() +conf = SparkConf(_jvm=gateway.jvm, _jconf=jconf) +sc = _zsc_ = SparkContext(jsc=jsc, gateway=gateway, conf=conf) + +if intp.isSpark2(): +from pyspark.sql import SparkSession + +spark = __zSpark__ = SparkSession(sc, intp.getSparkSession()) +sqlContext = sqlc = __zSqlc__ = __zSpark__._wrapped +else: +sqlContext = sqlc = __zSqlc__ = SQLContext(sparkContext=sc, sqlContext=intp.getSQLContext()) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/32517c9d/spark/src/main/sparkr-resources/interpreter-setting.json -- diff --git a/spark/src/main/sparkr-resources/interpreter-setting.json b/spark/src/main/sparkr-resources/interpreter-setting.json index d0fbd3e..300aff0 100644 --- a/spark/src/main/sparkr-resources/interpreter-setting.json +++ b/spark/src/main/sparkr-resources/interpreter-setting.json @@ -189,5 +189,16 @@ "editor": { "language": "r" } + }, + + { +"group": "spark", +"name": "ipyspark", +"className": "org.apache.zeppelin.spark.IPySparkInterpreter", +"properties": {}, +"editor": { + "language": "python", + "editOnDblClick": false +} } ] http://git-wip-us.apache.org/repos/asf/zeppelin/blob/32517c9d/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java -- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java new file mode 100644 index 000..5a2e884 --- /dev/null +++ b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.spark; + + +import com.google.common.io.Files; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apach
[02/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java deleted file mode 100644 index 95235e5..000 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ /dev/null @@ -1,975 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.interpreter.remote; - -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - -import org.apache.thrift.transport.TTransportException; -import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB; -import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Job.Status; -import org.apache.zeppelin.scheduler.Scheduler; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -public class RemoteInterpreterTest { - - - private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; - - private InterpreterGroup intpGroup; - private HashMap env; - - @Before - public void setUp() throws Exception { -intpGroup = new InterpreterGroup(); -env = new HashMap<>(); -env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); - } - - @After - public void tearDown() throws Exception { -intpGroup.close(); - } - - private RemoteInterpreter createMockInterpreterA(Properties p) { -return createMockInterpreterA(p, "note"); - } - - private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { -return new RemoteInterpreter( -p, -noteId, -MockInterpreterA.class.getName(), -new File(INTERPRETER_SCRIPT).getAbsolutePath(), -"fake", -"fakeRepo", -env, -10 * 1000, -null, -null, -"anonymous", -false); - } - - private RemoteInterpreter createMockInterpreterB(Properties p) { -return createMockInterpreterB(p, "note"); - } - - private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { -return new RemoteInterpreter( -p, -noteId, -MockInterpreterB.class.getName(), -new File(INTERPRETER_SCRIPT).getAbsolutePath(), -"fake", -"fakeRepo", -env, -10 * 1000, -null, -null, -"anonymous", -false); - } - - @Test - public void testRemoteInterperterCall() throws TTransportException, IOException { -Properties p = new Properties(); -intpGroup.put("note", new LinkedList()); - -RemoteInterpreter intpA = createMockInterpreterA(p); - -intpGroup.get("note").add(intpA); - -intpA.setInterpreterGroup(intpGroup); - -RemoteInterpreter intpB = createMockInterpreterB(p); - -intpGroup.get("note").add(intpB); -
[04/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
entListener { List getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { - return interpreterSettingManager.getInterpreters(note.getId()); + return interpreterSettingManager.getInterpreterBinding(note.getId()); } else { return new LinkedList<>(); } @@ -313,9 +312,10 @@ public class Notebook implements NoteEventListener { } public void moveNoteToTrash(String noteId) { -for (InterpreterSetting interpreterSetting : interpreterSettingManager -.getInterpreterSettings(noteId)) { - interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId); +try { + interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList()); +} catch (IOException e) { + e.printStackTrace(); } } @@ -339,7 +339,7 @@ public class Notebook implements NoteEventListener { // remove from all interpreter instance's angular object registry for (InterpreterSetting settings : interpreterSettingManager.get()) { AngularObjectRegistry registry = - settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); + settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { @@ -374,7 +374,7 @@ public class Notebook implements NoteEventListener { } } -ResourcePoolUtils.removeResourcesBelongsToNote(id); +interpreterSettingManager.removeResourcesBelongsToNote(id); fireNoteRemoveEvent(note); @@ -521,7 +521,8 @@ public class Notebook implements NoteEventListener { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); List settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { -InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId()); +InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(), +note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); String noteId = snapshot.getAngularObject().getNoteId(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 37138e6..bfe4566 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -93,10 +93,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { // since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph // see ZEPPELIN-212 - Object results; + volatile Object results; // For backward compatibility of note.json format after ZEPPELIN-212 - Object result; + volatile Object result; private Map runtimeInfos; /** @@ -157,7 +157,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } @Override - public void setResult(Object results) { + public synchronized void setResult(Object results) { this.results = results; } @@ -354,7 +354,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } @Override - public Object getReturn() { + public synchronized Object getReturn() { return results; } @@ -401,6 +401,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { logger.error("Can not find interpreter name " + repl); throw new RuntimeException("Can not find interpreter for " + getRequiredReplName()); } +//TODO(zjffdu) check interpreter setting status in interpreter setting itself InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId()); while (intp.getStatus().equals( org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { @@ -560,8 +561,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { InterpreterSetting intpGroup = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); - registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); - resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); + registry = intpGroup.getOrCreateInterpreterGroup(getUse
[03/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java deleted file mode 100644 index 1aab757..000 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java +++ /dev/null @@ -1,327 +0,0 @@ -package org.apache.zeppelin.interpreter; - -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; - -import org.junit.Test; - -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; - -public class InterpreterSettingTest { - - @Test - public void sharedModeCloseandRemoveInterpreterGroupTest() { -InterpreterOption interpreterOption = new InterpreterOption(); -interpreterOption.setPerUser(InterpreterOption.SHARED); -InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList(), new Properties(), new ArrayList(), interpreterOption, "", null); - -interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() { - @Override - public InterpreterGroup createInterpreterGroup(String interpreterGroupId, - InterpreterOption option) { -return new InterpreterGroup(interpreterGroupId); - } -}); - -Interpreter mockInterpreter1 = mock(RemoteInterpreter.class); -List interpreterList1 = new ArrayList<>(); -interpreterList1.add(mockInterpreter1); -InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1); - -// This won't effect anything -Interpreter mockInterpreter2 = mock(RemoteInterpreter.class); -List interpreterList2 = new ArrayList<>(); -interpreterList2.add(mockInterpreter2); -interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2); - -assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size()); - -interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2"); -assertEquals(0, interpreterSetting.getAllInterpreterGroups().size()); - } - - @Test - public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() { -InterpreterOption interpreterOption = new InterpreterOption(); -interpreterOption.setPerUser(InterpreterOption.SCOPED); -InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList(), new Properties(), new ArrayList(), interpreterOption, "", null); - -interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() { - @Override - public InterpreterGroup createInterpreterGroup(String interpreterGroupId, - InterpreterOption option) { -return new InterpreterGroup(interpreterGroupId); - } -}); - -Interpreter mockInterpreter1 = mock(RemoteInterpreter.class); -List interpreterList1 = new ArrayList<>(); -interpreterList1.add(mockInterpreter1); -InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1); - -Interpreter mockInterpreter2 = mock(RemoteInterpreter.class); -List interpreterList2 = new ArrayList<>(); -interpreterList2.add(mockInterpreter2); -interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2); - -assertEquals(1, interpreterSetting.getAllInterpreterGroups().size()); -assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size()); -assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size()); - -interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1"); -assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size()); - -// Check if non-existed key works or not -interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1"); -assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size()); - -interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2"); -assertEquals(0, interpreterSetting.getAllInterpreterGroups().size()); - } - - @Test - public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() { -InterpreterOption interpreterOption = new InterpreterOption(
[05/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java deleted file mode 100644 index 752b4e2..000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ /dev/null @@ -1,459 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.interpreter; - -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.zeppelin.dep.Dependency; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.annotations.SerializedName; -import com.google.gson.internal.StringMap; - -import static org.apache.zeppelin.notebook.utility.IdHashes.generateId; - -/** - * Interpreter settings - */ -public class InterpreterSetting { - - private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class); - private static final String SHARED_PROCESS = "shared_process"; - private String id; - private String name; - // always be null in case of InterpreterSettingRef - private String group; - private transient Map infos; - - // Map of the note and paragraphs which has runtime infos generated by this interpreter setting. - // This map is used to clear the infos in paragraph when the interpretersetting is restarted - private transient Map> runtimeInfosToBeCleared; - - /** - * properties can be either Map or - * Map - * properties should be: - * - Map when Interpreter instances are saved to - * `conf/interpreter.json` file - * - Map when Interpreters are registered - * : this is needed after https://github.com/apache/zeppelin/pull/1145 - * which changed the way of getting default interpreter setting AKA interpreterSettingsRef - */ - private Object properties; - private Status status; - private String errorReason; - - @SerializedName("interpreterGroup") - private List interpreterInfos; - private final transient Map interpreterGroupRef = new HashMap<>(); - private List dependencies = new LinkedList<>(); - private InterpreterOption option; - private transient String path; - - @SerializedName("runner") - private InterpreterRunner interpreterRunner; - - @Deprecated - private transient InterpreterGroupFactory interpreterGroupFactory; - - private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock; - private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock; - - public InterpreterSetting() { -ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); -interpreterGroupReadLock = lock.readLock(); -interpreterGroupWriteLock = lock.writeLock(); - } - - public InterpreterSetting(String id, String name, String group, - List interpreterInfos, Object properties, List dependencies, - InterpreterOption option, String path, InterpreterRunner runner) { -this(); -this.id = id; -this.name = name; -this.group = group; -this.interpreterInfos = interpreterInfos; -this.properties = properties; -this.dependencies = dependencies; -this.option = option; -this.path = path; -this.status = Status.READY; -this.interpreterRunner = runner; - } - - public InterpreterSetting(String name, String group, List interpreterInfos, - Object properties, List dependencies, InterpreterOption option, String path, - InterpreterRunner runner) { -this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path, -runner); - } - - /** - * Create interpreter from interpreterSettingRef - * -
[08/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java new file mode 100644 index 000..21d7526 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java @@ -0,0 +1,74 @@ +package org.apache.zeppelin.interpreter; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.junit.After; +import org.junit.Before; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +import static org.mockito.Mockito.mock; + + +/** + * This class will load configuration files under + * src/test/resources/interpreter + * src/test/resources/conf + * + * to construct InterpreterSettingManager and InterpreterFactory properly + * + */ +public abstract class AbstractInterpreterTest { + protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class); + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + + protected InterpreterSettingManager interpreterSettingManager; + protected InterpreterFactory interpreterFactory; + protected File testRootDir; + protected File interpreterDir; + protected File confDir; + protected File notebookDir; + protected ZeppelinConfiguration conf; + + @Before + public void setUp() throws Exception { +// copy the resources files to a temp folder +testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis()); +testRootDir.mkdirs(); +LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath()); +interpreterDir = new File(testRootDir, "interpreter"); +confDir = new File(testRootDir, "conf"); +notebookDir = new File(testRootDir, "notebook"); + +interpreterDir.mkdirs(); +confDir.mkdirs(); +notebookDir.mkdirs(); + +FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir); +FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir); + + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT); + +conf = new ZeppelinConfiguration(); +interpreterSettingManager = new InterpreterSettingManager(conf, +mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); +interpreterFactory = new InterpreterFactory(interpreterSettingManager); + } + + @After + public void tearDown() throws Exception { +interpreterSettingManager.close(); +FileUtils.deleteDirectory(testRootDir); + } +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java new file mode 100644 index 000..8eea4b2 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASI
[01/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
Repository: zeppelin Updated Branches: refs/heads/master 32517c9d9 -> 8d4902e71 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index e1a20b5..603be2e 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,31 +17,27 @@ package org.apache.zeppelin.notebook; -import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; - -import com.google.common.collect.Maps; -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.AbstractInterpreterTest; +import org.apache.zeppelin.interpreter.ClassloaderInterpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -56,18 +52,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.RepositoryException; -public class NotebookTest implements JobListenerFactory{ +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory { private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); - private File tmpDir; - private ZeppelinConfiguration conf; private SchedulerFactory schedulerFactory; - private File notebookDir; private Notebook notebook; private NotebookRepo notebookRepo; - private InterpreterFactory factory; - private InterpreterSettingManager interpreterSettingManager; - private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS; @@ -75,57 +88,30 @@ public class NotebookTest implements JobListenerFactory{ @Before public void setUp() throws Exception { +System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); +System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); +super.setUp(); -tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); -tmpDir.mkdirs(); -new File(tmpDir, "conf").mkdirs(); -notebookDir = new File(tmpDir + "/notebook"); -notebookDir.mkdirs(); - -System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf"); -System.setProperty(ConfVars.ZEPPELIN_HOME.get
[06/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java -- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 3ddeec0..102ca1a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -41,12 +41,7 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.helium.HeliumPackage; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterContextRunner; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResultMessage; -import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -463,7 +458,8 @@ public class NotebookServer extends WebSocketServlet Notebook notebook = notebook(); List notes = notebook.getAllNotes(); for (Note note : notes) { - List ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId()); + List ids = notebook.getInterpreterSettingManager() + .getInterpreterBinding(note.getId()); for (String id : ids) { if (id.equals(interpreterGroupId)) { broadcast(note.getId(), m); @@ -1003,7 +999,7 @@ public class NotebookServer extends WebSocketServlet List interpreterSettingIds = new LinkedList<>(); interpreterSettingIds.add(defaultInterpreterId); for (String interpreterSettingId : notebook.getInterpreterSettingManager(). -getDefaultInterpreterSettingList()) { +getInterpreterSettingIds()) { if (!interpreterSettingId.equals(defaultInterpreterId)) { interpreterSettingIds.add(interpreterSettingId); } @@ -1363,12 +1359,13 @@ public class NotebookServer extends WebSocketServlet List settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { -if (setting.getInterpreterGroup(user, note.getId()) == null) { +if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) { continue; } -if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) { +if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId()) +.getId())) { AngularObjectRegistry angularObjectRegistry = - setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); + setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); @@ -1405,12 +1402,13 @@ public class NotebookServer extends WebSocketServlet List settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getInterpreterGroup(user, n.getId()) == null) { + if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) { continue; } - if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) { + if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId()) + .getId())) { AngularObjectRegistry angularObjectRegistry = -setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry(); +setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry(); this.broadcastExcept(n.getId(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) .put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId()) @@ -2283,13 +2281,13 @@ public class NotebookServer extends WebSocketServlet for (InterpreterSetting intpSetting : settings) { AngularObjectRegistry registry = - intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); + intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); List objects = re
[11/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
[ZEPPELIN-2627] Interpreter refactor ### What is this PR for? I didn't intended to make such large change at the beginning, but found many things are coupled together that I have to make such large change. Several suggestions for you how to review and read it. * I move the interpreter package from zeppelin-zengine to zeppelin-interpreter, this is needed for this refactoring. * The overall change is the same as I described in the design doc. I would suggest you to read the unit test first. These unit test is very readable and easy to understand what the code is doing now. `InterpreterFactoryTest`, `InterpreterGroupTest`, `InterpreterSettingTest`, `InterpreterSettingManagerTest`, `RemoteInterpreterTest`. * Remove the referent counting logic. Now I will kill the interpreter process as long as all the sessions in the same interpreter group is closed. (I plan to add another kind of policy for the interpreter process lifecycle control, ZEPPELIN-2197) * The `RemoteFunction` I introduced is for reducing code duplicates when we use RPC. * The changes in Job.java and RemoteScheduler is for fixing the race issue bug. This bug cause the flaky test we see often in `ZeppelinSparkClusterTest.pySparkTest` ### What type of PR is it? [Bug Fix | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2627 ### How should this be tested? Unit test is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2422 from zjffdu/interpreter_refactor and squashes the following commits: 4724c98 [Jeff Zhang] [ZEPPELIN-2627] Interpreter Component Refactoring Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/8d4902e7 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/8d4902e7 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/8d4902e7 Branch: refs/heads/master Commit: 8d4902e717ba2932ddb0f7110f2a669811f5d1af Parents: 32517c9 Author: Jeff Zhang Authored: Tue May 23 15:52:15 2017 +0800 Committer: Jeff Zhang Committed: Mon Aug 28 09:14:11 2017 +0800 -- .travis.yml |1 + .../zeppelin/helium/ZeppelinDevServer.java |8 +- zeppelin-interpreter/pom.xml| 22 + .../zeppelin/conf/ZeppelinConfiguration.java| 835 + .../zeppelin/interpreter/Interpreter.java |1 - .../interpreter/InterpreterFactory.java | 115 ++ .../zeppelin/interpreter/InterpreterGroup.java | 272 ++--- .../zeppelin/interpreter/InterpreterInfo.java | 82 ++ .../interpreter/InterpreterInfoSaving.java | 101 ++ .../interpreter/InterpreterProperty.java|1 + .../zeppelin/interpreter/InterpreterRunner.java | 11 + .../interpreter/InterpreterSetting.java | 911 ++ .../interpreter/InterpreterSettingManager.java | 886 ++ .../interpreter/install/InstallInterpreter.java | 288 + .../remote/RemoteAngularObjectRegistry.java | 125 ++ .../interpreter/remote/RemoteInterpreter.java | 371 ++ .../remote/RemoteInterpreterEventPoller.java| 279 ++--- .../remote/RemoteInterpreterManagedProcess.java | 260 .../remote/RemoteInterpreterProcess.java| 154 +-- .../remote/RemoteInterpreterRunningProcess.java | 67 ++ .../remote/RemoteInterpreterServer.java | 58 +- .../zeppelin/resource/ResourcePoolUtils.java| 138 --- .../java/org/apache/zeppelin/scheduler/Job.java | 11 +- .../zeppelin/scheduler/RemoteScheduler.java | 134 +-- .../zeppelin/scheduler/SchedulerFactory.java| 49 +- .../zeppelin/tabledata/TableDataProxy.java |1 - .../java/org/apache/zeppelin/util/IdHashes.java | 76 ++ .../java/org/apache/zeppelin/util/Util.java | 76 ++ .../interpreter/AbstractInterpreterTest.java| 74 ++ .../interpreter/DoubleEchoInterpreter.java | 61 + .../zeppelin/interpreter/DummyInterpreter.java | 43 - .../zeppelin/interpreter/EchoInterpreter.java | 65 + .../interpreter/InterpreterFactoryTest.java | 66 + .../interpreter/InterpreterGroupTest.java | 90 ++ .../InterpreterOutputChangeWatcherTest.java | 11 +- .../InterpreterSettingManagerTest.java | 270 + .../interpreter/InterpreterSettingTest.java | 411 +++ .../zeppelin/interpreter/InterpreterTest.java |7 +- .../zeppelin/interpreter/SleepInterpreter.java | 60 + .../install/InstallInterpreterTest.java | 86 ++ .../interpreter/mock/MockInterpreter1.java | 106 ++ .../interpreter/mock/MockInterpreter11.java | 85 ++ .../interpreter/mock/MockInterpreter2.java | 104 ++ .../remote/AppendOutputRunnerTest.java |
[07/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java new file mode 100644 index 000..61e4ef0 --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class RemoteAngularObjectTest implements AngularObjectRegistryListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + + private RemoteInterpreter intp; + private InterpreterContext context; + private RemoteAngularObjectRegistry localRegistry; + private InterpreterSetting interpreterSetting; + + private AtomicInteger onAdd; + private AtomicInteger onUpdate; + private AtomicInteger onRemove; + + @Before + public void setUp() throws Exception { +onAdd = new AtomicInteger(0); +onUpdate = new AtomicInteger(0); +onRemove = new AtomicInteger(0); + +InterpreterOption interpreterOption = new InterpreterOption(); +interpreterOption.setRemote(true); +InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap()); +List interpreterInfos = new ArrayList<>(); +interpreterInfos.add(interpreterInfo1); +InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT); +interpreterSetting = new InterpreterSetting.Builder() +.setId("test") +.setName("test") +.setGroup("test") +.setInterpreterInfos(interpreterInfos) +.setOption(interpreterOption) +.setRunner(runner) +.setInterpreterDir("../interpeters/test") +.create(); + +intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); +localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry(); + +context = new InterpreterContext( +"note", +"id", +null, +"title", +"text", +new AuthenticationInfo(), +new HashMap(), +new GUI(), +new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null), +new LocalResourcePool("pool1"), +new LinkedList(), null); + +intp.open(); + + } + + @After + public void tearDown() throws Exception { +interpreterSetting.close(); + } + + @Test + public void testAngularObjectInterpreterSideCRUD() throws InterruptedException { +InterpreterResult ret = intp.interpret("get", context); +Thread.sleep(500); // waitFor eventpoller pool event +String[] result = ret.message().get(0).getData().split(" "); +assertEquals("0", result[0]); // size of registry +assertEquals("0", result[1]); // num watcher called + +// create object +ret = intp.interpret("add n1 v1", context); +Thread.sleep(500); +result = ret.message().get(0).getData().split(" "); +assert
[09/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
all(Client client) throws Exception { +client.cancel(sessionId, className, convert(context)); +return null; + } +}); + } + + @Override + public FormType getFormType() { +if (formType != null) { + return formType; +} + +// it is possible to call getFormType before it is opened +synchronized (this) { + if (!isOpened) { +open(); + } +} +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +FormType type = interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public FormType call(Client client) throws Exception { +formType = FormType.valueOf(client.getFormType(sessionId, className)); +return formType; + } +}); +return type; + } + + @Override + public int getProgress(final InterpreterContext context) { +if (!isOpened) { + LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className); + return 0; +} +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public Integer call(Client client) throws Exception { +return client.getProgress(sessionId, className, convert(context)); + } +}); + } + + + @Override + public List completion(final String buf, final int cursor, +final InterpreterContext interpreterContext) { +if (!isOpened) { + LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className); + return new ArrayList<>(); +} +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction>() { + @Override + public List call(Client client) throws Exception { +return client.completion(sessionId, className, buf, cursor, +convert(interpreterContext)); + } +}); + } + + public String getStatus(final String jobId) { +if (!isOpened) { + LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className); + return Job.Status.UNKNOWN.name(); +} +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public String call(Client client) throws Exception { +return client.getStatus(sessionId, jobId); + } +}); + } + + //TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ? + @Override + public Scheduler getScheduler() { +int maxConcurrency = Integer.parseInt( +property.getProperty("zeppelin.interpreter.max.poolsize", + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + "")); +return SchedulerFactory.singleton().createOrGetRemoteScheduler( +RemoteInterpreter.class.getName() + "-" + sessionId, +sessionId, this, maxConcurrency); + } + + private RemoteInterpreterContext convert(InterpreterContext ic) { +return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(), +ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()), +gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners())); + } + + private InterpreterResult convert(RemoteInterpreterResult result) { +InterpreterResult r = new InterpreterResult( +InterpreterResult.Code.valueOf(result.getCode())); + +for (RemoteInterpreterResultMessage m : result.getMsg()) { + r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData()); +} + +return r; + } + + /** + * Push local angular object registry to + * remote interpreter. This method should be + * call ONLY once when the first Interpreter is created + */ + private void pushAngularObjectRegistryToRemote(Client client) throws TException { +final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup() +.getAngularObjectRegistry(); +if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) { + final Map> registry = angularObjectRegistry + .getRegistry(); + LOGGER.info("Push local angular object registry from ZeppelinServer to" + + " remote interpreter group {}", this.getInterpreterGroup().getId()); + final java.lang.reflect.Type registryType = new TypeToken>>() { + }.getType(); + client.angul
[10/11] zeppelin git commit: [ZEPPELIN-2627] Interpreter refactor
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/8d4902e7/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java -- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java new file mode 100644 index 000..3f84cd0 --- /dev/null +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -0,0 +1,911 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.annotations.SerializedName; +import com.google.gson.internal.StringMap; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.remote.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; +import static org.apache.zeppelin.util.IdHashes.generateId; + +/** + * Represent one InterpreterSetting in the interpreter setting page + */ +public class InterpreterSetting { + + private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class); + private static final String SHARED_PROCESS = "shared_process"; + private static final String SHARED_SESSION = "shared_session"; + private static final Map DEFAULT_EDITOR = ImmutableMap.of( + "language", (Object) "text", + "editOnDblClick", false); + + private String id; + private String name; + // the original interpreter setting template name where it is created from + private String group; + + //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json + /** + * properties can be either Properties or Map + * properties should be: + * - Properties when Interpreter instances are saved to `conf/interpreter.json` file + * - Map when Interpreters are registered + * : this is needed after https://github.com/apache/zeppelin/pull/1145 + * which changed the way of getting default interpreter setting AKA interpreterSettingsRef + * Note(mina): In order to simplify the implementation, I chose to change properties + * from Properties to Object instead of creating new classes. + */ + private Object properties = new Properties(); + + private Status status; + private String errorReason; + + @SerializedName("interpreterGroup") + private List interpreterInfos; + + private List dependencies = new ArrayList<>(); + private InterpreterOption option = new InterpreterOption(true); + + @SerializedName("runner") + private InterpreterRunner interpreterRunner; + + /// + private transient InterpreterSettingManager interpreterSettingManager; + private transient String interpreterDir; + private final transient Map interpreterGroups = + new ConcurrentHas
[03/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java new file mode 100644 index 000..1aab757 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java @@ -0,0 +1,327 @@ +package org.apache.zeppelin.interpreter; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +import org.junit.Test; + +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +public class InterpreterSettingTest { + + @Test + public void sharedModeCloseandRemoveInterpreterGroupTest() { +InterpreterOption interpreterOption = new InterpreterOption(); +interpreterOption.setPerUser(InterpreterOption.SHARED); +InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList(), new Properties(), new ArrayList(), interpreterOption, "", null); + +interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() { + @Override + public InterpreterGroup createInterpreterGroup(String interpreterGroupId, + InterpreterOption option) { +return new InterpreterGroup(interpreterGroupId); + } +}); + +Interpreter mockInterpreter1 = mock(RemoteInterpreter.class); +List interpreterList1 = new ArrayList<>(); +interpreterList1.add(mockInterpreter1); +InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1"); +interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1); + +// This won't effect anything +Interpreter mockInterpreter2 = mock(RemoteInterpreter.class); +List interpreterList2 = new ArrayList<>(); +interpreterList2.add(mockInterpreter2); +interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1"); +interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2); + +assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size()); + +interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2"); +assertEquals(0, interpreterSetting.getAllInterpreterGroups().size()); + } + + @Test + public void perUserScopedModeCloseAndRemoveInterpreterGroupTest() { +InterpreterOption interpreterOption = new InterpreterOption(); +interpreterOption.setPerUser(InterpreterOption.SCOPED); +InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList(), new Properties(), new ArrayList(), interpreterOption, "", null); + +interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() { + @Override + public InterpreterGroup createInterpreterGroup(String interpreterGroupId, + InterpreterOption option) { +return new InterpreterGroup(interpreterGroupId); + } +}); + +Interpreter mockInterpreter1 = mock(RemoteInterpreter.class); +List interpreterList1 = new ArrayList<>(); +interpreterList1.add(mockInterpreter1); +InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1"); +interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1); + +Interpreter mockInterpreter2 = mock(RemoteInterpreter.class); +List interpreterList2 = new ArrayList<>(); +interpreterList2.add(mockInterpreter2); +interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1"); +interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2); + +assertEquals(1, interpreterSetting.getAllInterpreterGroups().size()); +assertEquals(2, interpreterSetting.getInterpreterGroup("user1", "note1").size()); +assertEquals(2, interpreterSetting.getInterpreterGroup("user2", "note1").size()); + +interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1"); +assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size()); + +// Check if non-existed key works or not +interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user1"); +assertEquals(1, interpreterSetting.getInterpreterGroup("user2","note1").size()); + +interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2"); +assertEquals(0, interpreterSetting.getAllInterpreterGroups().size()); + } + + @Test + public void perUserIsolatedModeCloseAndRemoveInterpreterGroupTest() { +InterpreterOption interpreterOption = new InterpreterOption(); +
[07/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java deleted file mode 100644 index 61e4ef0..000 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.interpreter.remote; - -import org.apache.zeppelin.display.AngularObject; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.display.GUI; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterAngular; -import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.user.AuthenticationInfo; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; - -public class RemoteAngularObjectTest implements AngularObjectRegistryListener { - private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; - - private RemoteInterpreter intp; - private InterpreterContext context; - private RemoteAngularObjectRegistry localRegistry; - private InterpreterSetting interpreterSetting; - - private AtomicInteger onAdd; - private AtomicInteger onUpdate; - private AtomicInteger onRemove; - - @Before - public void setUp() throws Exception { -onAdd = new AtomicInteger(0); -onUpdate = new AtomicInteger(0); -onRemove = new AtomicInteger(0); - -InterpreterOption interpreterOption = new InterpreterOption(); -interpreterOption.setRemote(true); -InterpreterInfo interpreterInfo1 = new InterpreterInfo(MockInterpreterAngular.class.getName(), "mock", true, new HashMap()); -List interpreterInfos = new ArrayList<>(); -interpreterInfos.add(interpreterInfo1); -InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT); -interpreterSetting = new InterpreterSetting.Builder() -.setId("test") -.setName("test") -.setGroup("test") -.setInterpreterInfos(interpreterInfos) -.setOption(interpreterOption) -.setRunner(runner) -.setInterpreterDir("../interpeters/test") -.create(); - -intp = (RemoteInterpreter) interpreterSetting.getDefaultInterpreter("user1", "note1"); -localRegistry = (RemoteAngularObjectRegistry) intp.getInterpreterGroup().getAngularObjectRegistry(); - -context = new InterpreterContext( -"note", -"id", -null, -"title", -"text", -new AuthenticationInfo(), -new HashMap(), -new GUI(), -new AngularObjectRegistry(intp.getInterpreterGroup().getId(), null), -new LocalResourcePool("pool1"), -new LinkedList(), null); - -intp.open(); - - } - - @After - public void tearDown() throws Exception { -interpreterSetting.close(); - } - - @Test - public void testAngularObjectInterpreterSideCRUD() throws InterruptedException { -InterpreterResult ret = intp.interpret("get", context); -Thread.sleep(500); // waitFor eventpoller pool event -String[] result = ret.message().get(0).getData().split(" "); -assertEquals("0", result[0]); // size of registry -assertEquals("0", result[1]); // num watcher called - -// create object -ret = intp.interpret("add n1 v1", context); -Thread.sleep(500); -result = ret.message().get(0).getData().split(" "); -as
[01/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
Repository: zeppelin Updated Branches: refs/heads/master 8d4902e71 -> 2a3791020 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 603be2e..e1a20b5 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,27 +17,31 @@ package org.apache.zeppelin.notebook; +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; + +import com.google.common.collect.Maps; +import java.io.File; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.AbstractInterpreterTest; -import org.apache.zeppelin.interpreter.ClassloaderInterpreter; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterFactory; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOption; -import org.apache.zeppelin.interpreter.InterpreterResult; -import org.apache.zeppelin.interpreter.InterpreterResultMessage; -import org.apache.zeppelin.interpreter.InterpreterSetting; -import org.apache.zeppelin.interpreter.LazyOpenInterpreter; +import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -52,35 +56,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.RepositoryException; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory { +public class NotebookTest implements JobListenerFactory{ private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); + private File tmpDir; + private ZeppelinConfiguration conf; private SchedulerFactory schedulerFactory; + private File notebookDir; private Notebook notebook; private NotebookRepo notebookRepo; + private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; + private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS; @@ -88,30 +75,57 @@ public class NotebookTest extends AbstractInterpreterTest implements JobListener @Before public void setUp() throws Exception { -System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); -System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); -super.setUp(); -schedulerFactory = SchedulerFactory.singleton(); +tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); +tmpDir.mkdirs(); +new File(tmpDir, "conf").mkdirs(); +notebookDir = new File(tmpDir + "/notebook"); +notebookDir.mkdirs(); + +System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), t
[09/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
all(Client client) throws Exception { -client.cancel(sessionId, className, convert(context)); -return null; - } -}); - } - - @Override - public FormType getFormType() { -if (formType != null) { - return formType; -} - -// it is possible to call getFormType before it is opened -synchronized (this) { - if (!isOpened) { -open(); - } -} -RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); -FormType type = interpreterProcess.callRemoteFunction( -new RemoteInterpreterProcess.RemoteFunction() { - @Override - public FormType call(Client client) throws Exception { -formType = FormType.valueOf(client.getFormType(sessionId, className)); -return formType; - } -}); -return type; - } - - @Override - public int getProgress(final InterpreterContext context) { -if (!isOpened) { - LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className); - return 0; -} -RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); -return interpreterProcess.callRemoteFunction( -new RemoteInterpreterProcess.RemoteFunction() { - @Override - public Integer call(Client client) throws Exception { -return client.getProgress(sessionId, className, convert(context)); - } -}); - } - - - @Override - public List completion(final String buf, final int cursor, -final InterpreterContext interpreterContext) { -if (!isOpened) { - LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className); - return new ArrayList<>(); -} -RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); -return interpreterProcess.callRemoteFunction( -new RemoteInterpreterProcess.RemoteFunction>() { - @Override - public List call(Client client) throws Exception { -return client.completion(sessionId, className, buf, cursor, -convert(interpreterContext)); - } -}); - } - - public String getStatus(final String jobId) { -if (!isOpened) { - LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className); - return Job.Status.UNKNOWN.name(); -} -RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); -return interpreterProcess.callRemoteFunction( -new RemoteInterpreterProcess.RemoteFunction() { - @Override - public String call(Client client) throws Exception { -return client.getStatus(sessionId, jobId); - } -}); - } - - //TODO(zjffdu) Share the Scheduler in the same session or in the same InterpreterGroup ? - @Override - public Scheduler getScheduler() { -int maxConcurrency = Integer.parseInt( -property.getProperty("zeppelin.interpreter.max.poolsize", - ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE.getIntValue() + "")); -return SchedulerFactory.singleton().createOrGetRemoteScheduler( -RemoteInterpreter.class.getName() + "-" + sessionId, -sessionId, this, maxConcurrency); - } - - private RemoteInterpreterContext convert(InterpreterContext ic) { -return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(), -ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()), -gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners())); - } - - private InterpreterResult convert(RemoteInterpreterResult result) { -InterpreterResult r = new InterpreterResult( -InterpreterResult.Code.valueOf(result.getCode())); - -for (RemoteInterpreterResultMessage m : result.getMsg()) { - r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData()); -} - -return r; - } - - /** - * Push local angular object registry to - * remote interpreter. This method should be - * call ONLY once when the first Interpreter is created - */ - private void pushAngularObjectRegistryToRemote(Client client) throws TException { -final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup() -.getAngularObjectRegistry(); -if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) { - final Map> registry = angularObjectRegistry - .getRegistry(); - LOGGER.info("Push local angular object registry from ZeppelinServer to" + - " remote interpreter group {}", this.getInterpreterGroup().getId()); - final java.lang.reflect.Type registryType = new TypeToken>>() { - }.getType(); - client.angul
[08/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java deleted file mode 100644 index 21d7526..000 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.apache.zeppelin.interpreter; - -import org.apache.commons.io.FileUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.junit.After; -import org.junit.Before; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; - -import static org.mockito.Mockito.mock; - - -/** - * This class will load configuration files under - * src/test/resources/interpreter - * src/test/resources/conf - * - * to construct InterpreterSettingManager and InterpreterFactory properly - * - */ -public abstract class AbstractInterpreterTest { - protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractInterpreterTest.class); - private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; - - protected InterpreterSettingManager interpreterSettingManager; - protected InterpreterFactory interpreterFactory; - protected File testRootDir; - protected File interpreterDir; - protected File confDir; - protected File notebookDir; - protected ZeppelinConfiguration conf; - - @Before - public void setUp() throws Exception { -// copy the resources files to a temp folder -testRootDir = new File(System.getProperty("java.io.tmpdir") + "/Zeppelin_Test_" + System.currentTimeMillis()); -testRootDir.mkdirs(); -LOGGER.info("Create tmp directory: {} as root folder of ZEPPELIN_INTERPRETER_DIR & ZEPPELIN_CONF_DIR", testRootDir.getAbsolutePath()); -interpreterDir = new File(testRootDir, "interpreter"); -confDir = new File(testRootDir, "conf"); -notebookDir = new File(testRootDir, "notebook"); - -interpreterDir.mkdirs(); -confDir.mkdirs(); -notebookDir.mkdirs(); - -FileUtils.copyDirectory(new File("src/test/resources/interpreter"), interpreterDir); -FileUtils.copyDirectory(new File("src/test/resources/conf"), confDir); - - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_CONF_DIR.getVarName(), confDir.getAbsolutePath()); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DIR.getVarName(), interpreterDir.getAbsolutePath()); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); - System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_REMOTE_RUNNER.getVarName(), INTERPRETER_SCRIPT); - -conf = new ZeppelinConfiguration(); -interpreterSettingManager = new InterpreterSettingManager(conf, -mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); -interpreterFactory = new InterpreterFactory(interpreterSettingManager); - } - - @After - public void tearDown() throws Exception { -interpreterSettingManager.close(); -FileUtils.deleteDirectory(testRootDir); - } -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java deleted file mode 100644 index 8eea4b2..000 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/DoubleEchoInterpreter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS
[02/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java new file mode 100644 index 000..95235e5 --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -0,0 +1,975 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import static org.junit.Assert.*; + +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.apache.thrift.transport.TTransportException; +import org.apache.zeppelin.display.AngularObject; +import org.apache.zeppelin.display.AngularObjectRegistry; +import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; +import org.apache.zeppelin.user.AuthenticationInfo; +import org.apache.zeppelin.display.GUI; +import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.InterpreterResult.Code; +import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; +import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB; +import org.apache.zeppelin.resource.LocalResourcePool; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.apache.zeppelin.scheduler.Scheduler; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +public class RemoteInterpreterTest { + + + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + + private InterpreterGroup intpGroup; + private HashMap env; + + @Before + public void setUp() throws Exception { +intpGroup = new InterpreterGroup(); +env = new HashMap<>(); +env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); + } + + @After + public void tearDown() throws Exception { +intpGroup.close(); + } + + private RemoteInterpreter createMockInterpreterA(Properties p) { +return createMockInterpreterA(p, "note"); + } + + private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { +return new RemoteInterpreter( +p, +noteId, +MockInterpreterA.class.getName(), +new File(INTERPRETER_SCRIPT).getAbsolutePath(), +"fake", +"fakeRepo", +env, +10 * 1000, +null, +null, +"anonymous", +false); + } + + private RemoteInterpreter createMockInterpreterB(Properties p) { +return createMockInterpreterB(p, "note"); + } + + private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { +return new RemoteInterpreter( +p, +noteId, +MockInterpreterB.class.getName(), +new File(INTERPRETER_SCRIPT).getAbsolutePath(), +"fake", +"fakeRepo", +env, +10 * 1000, +null, +null, +"anonymous", +false); + } + + @Test + public void testRemoteInterperterCall() throws TTransportException, IOException { +Properties p = new Properties(); +intpGroup.put("note", new LinkedList()); + +RemoteInterpreter intpA = createMockInterpreterA(p); + +intpGroup.get("note").add(intpA); + +intpA.setInterpreterGroup(intpGroup); + +RemoteInterpreter intpB = createMockInterpreterB(p); + +intpGroup.get("note").add(intpB); +intp
[06/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java -- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 102ca1a..3ddeec0 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -41,7 +41,12 @@ import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.helium.HeliumPackage; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterContextRunner; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; @@ -458,8 +463,7 @@ public class NotebookServer extends WebSocketServlet Notebook notebook = notebook(); List notes = notebook.getAllNotes(); for (Note note : notes) { - List ids = notebook.getInterpreterSettingManager() - .getInterpreterBinding(note.getId()); + List ids = notebook.getInterpreterSettingManager().getInterpreters(note.getId()); for (String id : ids) { if (id.equals(interpreterGroupId)) { broadcast(note.getId(), m); @@ -999,7 +1003,7 @@ public class NotebookServer extends WebSocketServlet List interpreterSettingIds = new LinkedList<>(); interpreterSettingIds.add(defaultInterpreterId); for (String interpreterSettingId : notebook.getInterpreterSettingManager(). -getInterpreterSettingIds()) { +getDefaultInterpreterSettingList()) { if (!interpreterSettingId.equals(defaultInterpreterId)) { interpreterSettingIds.add(interpreterSettingId); } @@ -1359,13 +1363,12 @@ public class NotebookServer extends WebSocketServlet List settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { -if (setting.getOrCreateInterpreterGroup(user, note.getId()) == null) { +if (setting.getInterpreterGroup(user, note.getId()) == null) { continue; } -if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, note.getId()) -.getId())) { +if (interpreterGroupId.equals(setting.getInterpreterGroup(user, note.getId()).getId())) { AngularObjectRegistry angularObjectRegistry = - setting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); + setting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); // first trying to get local registry ao = angularObjectRegistry.get(varName, noteId, paragraphId); @@ -1402,13 +1405,12 @@ public class NotebookServer extends WebSocketServlet List settings = notebook.getInterpreterSettingManager().getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { - if (setting.getOrCreateInterpreterGroup(user, n.getId()) == null) { + if (setting.getInterpreterGroup(user, n.getId()) == null) { continue; } - if (interpreterGroupId.equals(setting.getOrCreateInterpreterGroup(user, n.getId()) - .getId())) { + if (interpreterGroupId.equals(setting.getInterpreterGroup(user, n.getId()).getId())) { AngularObjectRegistry angularObjectRegistry = -setting.getOrCreateInterpreterGroup(user, n.getId()).getAngularObjectRegistry(); +setting.getInterpreterGroup(user, n.getId()).getAngularObjectRegistry(); this.broadcastExcept(n.getId(), new Message(OP.ANGULAR_OBJECT_UPDATE).put("angularObject", ao) .put("interpreterGroupId", interpreterGroupId).put("noteId", n.getId()) @@ -2281,13 +2283,13 @@ public class NotebookServer extends WebSocketServlet for (InterpreterSetting intpSetting : settings) { AngularObjectRegistry registry = - intpSetting.getOrCreateInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); + intpSetting.getInterpreterGroup(user, note.getId()).getAngularObjectRegistry(); List objects = re
[11/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
Revert "[ZEPPELIN-2627] Interpreter refactor" This reverts commit 8d4902e717ba2932ddb0f7110f2a669811f5d1af. ### What is this PR for? Just revert ZEPPELIN-2627 ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2627 ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2553 from zjffdu/Revert_2627 and squashes the following commits: 222d957 [Jeff Zhang] Revert "[ZEPPELIN-2627] Interpreter refactor" Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/2a379102 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/2a379102 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/2a379102 Branch: refs/heads/master Commit: 2a3791020fc74700b3310bef5dea2bb6842d9de1 Parents: 8d4902e Author: Jeff Zhang Authored: Mon Aug 28 13:51:57 2017 +0800 Committer: Jeff Zhang Committed: Mon Aug 28 14:49:42 2017 +0800 -- .travis.yml |1 - .../zeppelin/helium/ZeppelinDevServer.java |8 +- zeppelin-interpreter/pom.xml| 22 - .../zeppelin/conf/ZeppelinConfiguration.java| 835 - .../zeppelin/interpreter/Interpreter.java |1 + .../interpreter/InterpreterFactory.java | 115 -- .../zeppelin/interpreter/InterpreterGroup.java | 272 +++-- .../zeppelin/interpreter/InterpreterInfo.java | 82 -- .../interpreter/InterpreterInfoSaving.java | 101 -- .../interpreter/InterpreterProperty.java|1 - .../zeppelin/interpreter/InterpreterRunner.java | 11 - .../interpreter/InterpreterSetting.java | 911 -- .../interpreter/InterpreterSettingManager.java | 886 -- .../interpreter/install/InstallInterpreter.java | 288 - .../remote/RemoteAngularObjectRegistry.java | 125 -- .../interpreter/remote/RemoteInterpreter.java | 371 -- .../remote/RemoteInterpreterEventPoller.java| 279 +++-- .../remote/RemoteInterpreterManagedProcess.java | 260 .../remote/RemoteInterpreterProcess.java| 154 ++- .../remote/RemoteInterpreterRunningProcess.java | 67 -- .../remote/RemoteInterpreterServer.java | 58 +- .../zeppelin/resource/ResourcePoolUtils.java| 138 +++ .../java/org/apache/zeppelin/scheduler/Job.java | 11 +- .../zeppelin/scheduler/RemoteScheduler.java | 134 ++- .../zeppelin/scheduler/SchedulerFactory.java| 49 +- .../zeppelin/tabledata/TableDataProxy.java |1 + .../java/org/apache/zeppelin/util/IdHashes.java | 76 -- .../java/org/apache/zeppelin/util/Util.java | 76 -- .../interpreter/AbstractInterpreterTest.java| 74 -- .../interpreter/DoubleEchoInterpreter.java | 61 - .../zeppelin/interpreter/DummyInterpreter.java | 43 + .../zeppelin/interpreter/EchoInterpreter.java | 65 - .../interpreter/InterpreterFactoryTest.java | 66 - .../interpreter/InterpreterGroupTest.java | 90 -- .../InterpreterOutputChangeWatcherTest.java | 11 +- .../InterpreterSettingManagerTest.java | 270 - .../interpreter/InterpreterSettingTest.java | 411 --- .../zeppelin/interpreter/InterpreterTest.java |7 +- .../zeppelin/interpreter/SleepInterpreter.java | 60 - .../install/InstallInterpreterTest.java | 86 -- .../interpreter/mock/MockInterpreter1.java | 106 -- .../interpreter/mock/MockInterpreter11.java | 85 -- .../interpreter/mock/MockInterpreter2.java | 104 -- .../remote/AppendOutputRunnerTest.java | 232 .../remote/RemoteAngularObjectTest.java | 196 --- .../RemoteInterpreterEventPollerTest.java | 55 - .../RemoteInterpreterOutputTestStream.java | 178 --- .../remote/RemoteInterpreterTest.java | 520 .../remote/RemoteInterpreterUtilsTest.java | 33 - .../remote/mock/GetEnvPropertyInterpreter.java | 82 -- .../remote/mock/MockInterpreterA.java | 96 -- .../remote/mock/MockInterpreterAngular.java | 112 -- .../remote/mock/MockInterpreterB.java | 120 -- .../mock/MockInterpreterOutputStream.java | 93 -- .../mock/MockInterpreterResourcePool.java | 125 -- .../zeppelin/scheduler/RemoteSchedulerTest.java | 343 -- .../src/test/resources/conf/interpreter.json| 115 -- .../interpreter/test/interpreter-setting.json | 42 - .../src/test/resources/log4j.properties |4 +- .../zeppelin/rest/InterpreterRestApi.java |4 +- .../apache/zeppelin/server/ZeppelinServer.java | 35 +- .../apache/zeppelin/socket/NotebookServer.java | 32 +- .../interpreter/mock/MockInterpreter1.java | 75 +
[04/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
r { List getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { - return interpreterSettingManager.getInterpreterBinding(note.getId()); + return interpreterSettingManager.getInterpreters(note.getId()); } else { return new LinkedList<>(); } @@ -312,10 +313,9 @@ public class Notebook implements NoteEventListener { } public void moveNoteToTrash(String noteId) { -try { - interpreterSettingManager.setInterpreterBinding("", noteId, new ArrayList()); -} catch (IOException e) { - e.printStackTrace(); +for (InterpreterSetting interpreterSetting : interpreterSettingManager +.getInterpreterSettings(noteId)) { + interpreterSettingManager.removeInterpretersForNote(interpreterSetting, "", noteId); } } @@ -339,7 +339,7 @@ public class Notebook implements NoteEventListener { // remove from all interpreter instance's angular object registry for (InterpreterSetting settings : interpreterSettingManager.get()) { AngularObjectRegistry registry = - settings.getOrCreateInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); + settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { // remove paragraph scope object for (Paragraph p : note.getParagraphs()) { @@ -374,7 +374,7 @@ public class Notebook implements NoteEventListener { } } -interpreterSettingManager.removeResourcesBelongsToNote(id); +ResourcePoolUtils.removeResourcesBelongsToNote(id); fireNoteRemoveEvent(note); @@ -521,8 +521,7 @@ public class Notebook implements NoteEventListener { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); List settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { -InterpreterGroup intpGroup = setting.getOrCreateInterpreterGroup(subject.getUser(), -note.getId()); +InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { AngularObjectRegistry registry = intpGroup.getAngularObjectRegistry(); String noteId = snapshot.getAngularObject().getNoteId(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index bfe4566..37138e6 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -93,10 +93,10 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { // since zeppelin-0.7.0, zeppelin stores multiple results of the paragraph // see ZEPPELIN-212 - volatile Object results; + Object results; // For backward compatibility of note.json format after ZEPPELIN-212 - volatile Object result; + Object result; private Map runtimeInfos; /** @@ -157,7 +157,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } @Override - public synchronized void setResult(Object results) { + public void setResult(Object results) { this.results = results; } @@ -354,7 +354,7 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { } @Override - public synchronized Object getReturn() { + public Object getReturn() { return results; } @@ -401,7 +401,6 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { logger.error("Can not find interpreter name " + repl); throw new RuntimeException("Can not find interpreter for " + getRequiredReplName()); } -//TODO(zjffdu) check interpreter setting status in interpreter setting itself InterpreterSetting intp = getInterpreterSettingById(repl.getInterpreterGroup().getId()); while (intp.getStatus().equals( org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { @@ -561,10 +560,8 @@ public class Paragraph extends Job implements Cloneable, JsonSerializable { if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { InterpreterSetting intpGroup = interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); - registry = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId()) - .getAngularObjectRegistry(); - resourcePool = intpGroup.getOrCreateInterpreterGroup(getUser(), note.getId()) - .getResourcePool(); + registry = intpGroup.get
[10/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java -- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java deleted file mode 100644 index 3f84cd0..000 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ /dev/null @@ -1,911 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - *http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.interpreter; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.annotations.SerializedName; -import com.google.gson.internal.StringMap; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.remote.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileNotFoundException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; -import static org.apache.zeppelin.util.IdHashes.generateId; - -/** - * Represent one InterpreterSetting in the interpreter setting page - */ -public class InterpreterSetting { - - private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class); - private static final String SHARED_PROCESS = "shared_process"; - private static final String SHARED_SESSION = "shared_session"; - private static final Map DEFAULT_EDITOR = ImmutableMap.of( - "language", (Object) "text", - "editOnDblClick", false); - - private String id; - private String name; - // the original interpreter setting template name where it is created from - private String group; - - //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json - /** - * properties can be either Properties or Map - * properties should be: - * - Properties when Interpreter instances are saved to `conf/interpreter.json` file - * - Map when Interpreters are registered - * : this is needed after https://github.com/apache/zeppelin/pull/1145 - * which changed the way of getting default interpreter setting AKA interpreterSettingsRef - * Note(mina): In order to simplify the implementation, I chose to change properties - * from Properties to Object instead of creating new classes. - */ - private Object properties = new Properties(); - - private Status status; - private String errorReason; - - @SerializedName("interpreterGroup") - private List interpreterInfos; - - private List dependencies = new ArrayList<>(); - private InterpreterOption option = new InterpreterOption(true); - - @SerializedName("runner") - private InterpreterRunner interpreterRunner; - - /// - private transient InterpreterSettingManager interpreterSettingManager; - private transient String interpreterDir; - private final transient Map interpreterGroups = - new ConcurrentHas
[05/11] zeppelin git commit: Revert "[ZEPPELIN-2627] Interpreter refactor"
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a379102/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java new file mode 100644 index 000..752b4e2 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -0,0 +1,459 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +import org.apache.zeppelin.dep.Dependency; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.annotations.SerializedName; +import com.google.gson.internal.StringMap; + +import static org.apache.zeppelin.notebook.utility.IdHashes.generateId; + +/** + * Interpreter settings + */ +public class InterpreterSetting { + + private static final Logger logger = LoggerFactory.getLogger(InterpreterSetting.class); + private static final String SHARED_PROCESS = "shared_process"; + private String id; + private String name; + // always be null in case of InterpreterSettingRef + private String group; + private transient Map infos; + + // Map of the note and paragraphs which has runtime infos generated by this interpreter setting. + // This map is used to clear the infos in paragraph when the interpretersetting is restarted + private transient Map> runtimeInfosToBeCleared; + + /** + * properties can be either Map or + * Map + * properties should be: + * - Map when Interpreter instances are saved to + * `conf/interpreter.json` file + * - Map when Interpreters are registered + * : this is needed after https://github.com/apache/zeppelin/pull/1145 + * which changed the way of getting default interpreter setting AKA interpreterSettingsRef + */ + private Object properties; + private Status status; + private String errorReason; + + @SerializedName("interpreterGroup") + private List interpreterInfos; + private final transient Map interpreterGroupRef = new HashMap<>(); + private List dependencies = new LinkedList<>(); + private InterpreterOption option; + private transient String path; + + @SerializedName("runner") + private InterpreterRunner interpreterRunner; + + @Deprecated + private transient InterpreterGroupFactory interpreterGroupFactory; + + private final transient ReentrantReadWriteLock.ReadLock interpreterGroupReadLock; + private final transient ReentrantReadWriteLock.WriteLock interpreterGroupWriteLock; + + public InterpreterSetting() { +ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); +interpreterGroupReadLock = lock.readLock(); +interpreterGroupWriteLock = lock.writeLock(); + } + + public InterpreterSetting(String id, String name, String group, + List interpreterInfos, Object properties, List dependencies, + InterpreterOption option, String path, InterpreterRunner runner) { +this(); +this.id = id; +this.name = name; +this.group = group; +this.interpreterInfos = interpreterInfos; +this.properties = properties; +this.dependencies = dependencies; +this.option = option; +this.path = path; +this.status = Status.READY; +this.interpreterRunner = runner; + } + + public InterpreterSetting(String name, String group, List interpreterInfos, + Object properties, List dependencies, InterpreterOption option, String path, + InterpreterRunner runner) { +this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path, +runner); + } + + /** + * Create interpreter from interpreterSettingRef + * + * @
[2/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 95235e5..0a5e484 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -17,839 +17,352 @@ package org.apache.zeppelin.interpreter.remote; -import static org.junit.Assert.*; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; - import org.apache.thrift.transport.TTransportException; -import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterEnv; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; -import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; -import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.InterpreterResult.Code; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterA; -import org.apache.zeppelin.interpreter.remote.mock.MockInterpreterB; -import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Job.Status; -import org.apache.zeppelin.scheduler.Scheduler; +import org.apache.zeppelin.interpreter.remote.mock.GetAngularObjectSizeInterpreter; +import org.apache.zeppelin.interpreter.remote.mock.GetEnvPropertyInterpreter; +import org.apache.zeppelin.user.AuthenticationInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Properties; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; public class RemoteInterpreterTest { private static final String INTERPRETER_SCRIPT = - System.getProperty("os.name").startsWith("Windows") ? - "../bin/interpreter.cmd" : - "../bin/interpreter.sh"; + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; - private InterpreterGroup intpGroup; - private HashMap env; + private InterpreterSetting interpreterSetting; @Before public void setUp() throws Exception { -intpGroup = new InterpreterGroup(); -env = new HashMap<>(); -env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); +InterpreterOption interpreterOption = new InterpreterOption(); + +interpreterOption.setRemote(true); +InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); +InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); +InterpreterInfo interpreterInfo3 = new InterpreterInfo(SleepInterpreter.class.getName(), "sleep", false, new HashMap()); +InterpreterInfo interpreterInfo4 = new InterpreterInfo(GetEnvPropertyInterpreter.class.getName(), "get", false, new HashMap()); +InterpreterInfo interpreterInfo5 = new InterpreterInfo(GetAngularObjectSizeInterpreter.class.getName(), "angular_obj",false, new HashMap()); +List interpreterInfos = new ArrayList<>(); +interpreterInfos.add(interpreterInfo1); +interpreterInfos.add(interpreterInfo2); +interpreterInfos.add(interpreterInfo3); +interpreterInfos.add(interpreterInfo4); +interpreterInfos.add(interpreterInfo5); +InterpreterRunner runner = new InterpreterRunner(INTERPRETER_SCRIPT, INTERPRETER_SCRIPT); +interpreterSetting = new InterpreterSetting.Builder() +.setId("test") +.setName("test") +.setGroup("test") +.setInterpreterInfos(interpreterInfos) +.setOption(interpreterOption) +.setRunner(runner) +.setInterpreterDir("../interpeters/test") +.create(); } @After public void tearDown() throws Exception { -intpGroup.close(); +interpreterSetting.close(); } - private RemoteInterpreter createMockInterpreterA(Properties p) { -return
[7/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 9403b4f..f020919 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -17,288 +17,31 @@ package org.apache.zeppelin.interpreter; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; -import org.apache.commons.lang.NullArgumentException; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.display.AngularObjectRegistryListener; -import org.apache.zeppelin.helium.ApplicationEventListener; -import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositoryException; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Properties; /** + * //TODO(zjffdu) considering to move to InterpreterSettingManager + * * Manage interpreters. */ -public class InterpreterFactory implements InterpreterGroupFactory { - private static final Logger logger = LoggerFactory.getLogger(InterpreterFactory.class); - - private Map cleanCl = - Collections.synchronizedMap(new HashMap()); - - private ZeppelinConfiguration conf; +public class InterpreterFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class); private final InterpreterSettingManager interpreterSettingManager; - private AngularObjectRegistryListener angularObjectRegistryListener; - private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; - private final ApplicationEventListener appEventListener; - - private boolean shiroEnabled; - - private Map env = new HashMap<>(); - - private Interpreter devInterpreter; - - public InterpreterFactory(ZeppelinConfiguration conf, - AngularObjectRegistryListener angularObjectRegistryListener, - RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, DependencyResolver depResolver, - boolean shiroEnabled, InterpreterSettingManager interpreterSettingManager) - throws InterpreterException, IOException, RepositoryException { -this.conf = conf; -this.angularObjectRegistryListener = angularObjectRegistryListener; -this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; -this.appEventListener = appEventListener; -this.shiroEnabled = shiroEnabled; + public InterpreterFactory(InterpreterSettingManager interpreterSettingManager) { this.interpreterSettingManager = interpreterSettingManager; -//TODO(jl): Fix it not to use InterpreterGroupFactory -interpreterSettingManager.setInterpreterGroupFactory(this); - -logger.info("shiroEnabled: {}", shiroEnabled); - } - - /** - * @param id interpreterGroup id. Combination of interpreterSettingId + noteId/userId/shared - * depends on interpreter mode - */ - @Override - public InterpreterGroup createInterpreterGroup(String id, InterpreterOption option) - throws InterpreterException, NullArgumentException { - -//When called from REST API without option we receive NPE -if (option == null) { - throw new NullArgumentException("option"); -} - -AngularObjectRegistry angularObjectRegistry; - -InterpreterGroup interpreterGroup = new InterpreterGroup(id); -if (option.isRemote()) { - angularObjectRegistry = - new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup); -} else { - angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener); - - // TODO(moon) : create distributed resource pool for local interpreters and set -} - -interpreterGroup.setAngularObjectRegistry(angularObjectRegistry); -
[3/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java index 1aab757..3c061a9 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingTest.java @@ -1,327 +1,411 @@ -package org.apache.zeppelin.interpreter; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -import java.util.ArrayList; -import java.util.List; -import java.util.Properties; +package org.apache.zeppelin.interpreter; import org.junit.Test; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; +import static org.junit.Assert.assertNull; public class InterpreterSettingTest { @Test - public void sharedModeCloseandRemoveInterpreterGroupTest() { + public void testCreateInterpreters() { InterpreterOption interpreterOption = new InterpreterOption(); interpreterOption.setPerUser(InterpreterOption.SHARED); -InterpreterSetting interpreterSetting = new InterpreterSetting("", "", "", new ArrayList(), new Properties(), new ArrayList(), interpreterOption, "", null); - -interpreterSetting.setInterpreterGroupFactory(new InterpreterGroupFactory() { - @Override - public InterpreterGroup createInterpreterGroup(String interpreterGroupId, - InterpreterOption option) { -return new InterpreterGroup(interpreterGroupId); - } -}); - -Interpreter mockInterpreter1 = mock(RemoteInterpreter.class); -List interpreterList1 = new ArrayList<>(); -interpreterList1.add(mockInterpreter1); -InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup("user1", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user1", "note1"), interpreterList1); - -// This won't effect anything -Interpreter mockInterpreter2 = mock(RemoteInterpreter.class); -List interpreterList2 = new ArrayList<>(); -interpreterList2.add(mockInterpreter2); -interpreterGroup = interpreterSetting.getInterpreterGroup("user2", "note1"); -interpreterGroup.put(interpreterSetting.getInterpreterSessionKey("user2", "note1"), interpreterList2); - -assertEquals(1, interpreterSetting.getInterpreterGroup("user1", "note1").size()); - -interpreterSetting.closeAndRemoveInterpreterGroup("note1", "user2"); -assertEquals(0, interpreterSetting.getAllInterpreterGroups().size()); +interpreterOption.setRemote(false); +InterpreterInfo interpreterInfo1 = new InterpreterInfo(EchoInterpreter.class.getName(), "echo", true, new HashMap()); +InterpreterInfo interpreterInfo2 = new InterpreterInfo(DoubleEchoInterpreter.class.getName(), "double_echo", false, new HashMap()); +List interpreterInfos = new ArrayList<>(); +interpreterInfos.add(interpreterInfo1); +interpreterInfos.add(interpreterInfo2); +InterpreterSetting interpreterSetting = new InterpreterSetting.Builder() +.setId("id") +.setName("test") +.setGroup("test") +.setInterpreterInfos(interpreterInfos) +.setOption(interpreterOption) +.create(); + +// create default interpreter for user1 and note1 +assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getDefaultInterpreter("user1", "note1").getClassName()); + +// create interpreter echo for user1 and note1 +assertEquals(EchoInterpreter.class.getName(), interpreterSetting.getInterpreter("user1", "note1", "echo").getClassName()); +assertEquals(interpreterSetting.getDefaultInterpreter("user1", "note1"), interpreterSetting.getInterpreter("user1", "note1", "echo")); + +// create interpreter
[6/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 12545d6..585a58a 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -17,212 +17,205 @@ package org.apache.zeppelin.interpreter; -import java.io.BufferedReader; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.display.AngularObjectRegistryListener; +import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; +import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; +import org.apache.zeppelin.resource.Resource; +import org.apache.zeppelin.resource.ResourcePool; +import org.apache.zeppelin.resource.ResourceSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; +import org.sonatype.aether.repository.RemoteRepository; + import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.OutputStreamWriter; import java.lang.reflect.Type; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; -import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream.Filter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; -import java.util.EnumSet; -import java.util.Enumeration; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; -import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; -import org.apache.zeppelin.scheduler.Job; -import org.apache.zeppelin.scheduler.Job.Status; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositoryException; -import org.sonatype.aether.repository.Authentication; -import org.sonatype.aether.repository.Proxy; -import org.sonatype.aether.repository.RemoteRepository; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Maps; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.internal.StringMap; -import com.google.gson.reflect.TypeToken; - -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; /** - * TBD + * InterpreterSettingManager is the component which manage all the interpreter settings. + * (load/create/update/remove/get) + * Besides that InterpreterSettingManager also manage the interpreter setting binding. + * TODO(zjffdu) We could move it into another separated component. */ public class InterpreterSettingManager { - private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class); - private static final String
[8/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
nts InterpreterOutputChan tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); tmpDir.mkdirs(); fileChanged = null; -numChanged = 0; +numChanged = new AtomicInteger(0); } @After @@ -66,7 +67,7 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan @Test public void test() throws IOException, InterruptedException { assertNull(fileChanged); -assertEquals(0, numChanged); +assertEquals(0, numChanged.get()); Thread.sleep(1000); // create new file @@ -92,14 +93,14 @@ public class InterpreterOutputChangeWatcherTest implements InterpreterOutputChan } assertNotNull(fileChanged); -assertEquals(1, numChanged); +assertEquals(1, numChanged.get()); } @Override public void fileChanged(File file) { fileChanged = file; -numChanged++; +numChanged.incrementAndGet(); synchronized(this) { notify(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java index 305268c..31c9225 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/InterpreterTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import static org.junit.Assert.assertEquals; +//TODO(zjffdu) add more test for Interpreter which is a very important class public class InterpreterTest { @Test @@ -85,4 +86,41 @@ public class InterpreterTest { ); } + public static class DummyInterpreter extends Interpreter { + +public DummyInterpreter(Properties property) { + super(property); +} + +@Override +public void open() { + +} + +@Override +public void close() { + +} + +@Override +public InterpreterResult interpret(String st, InterpreterContext context) { + return null; +} + +@Override +public void cancel(InterpreterContext context) { + +} + +@Override +public FormType getFormType() { + return null; +} + +@Override +public int getProgress(InterpreterContext context) { + return 0; +} + } + } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java -- diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java new file mode 100644 index 000..5f7426a --- /dev/null +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterUtilsTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter.remote; + +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertTrue; + +public class RemoteInterpreterUtilsTest { + + @Test + public void testFindRandomAvailablePortOnAllLocalInterfaces() throws IOException { + assertTrue(RemoteInterpreterUtils.findRandomAvailablePortOnAllLocalInterfaces() > 0); + } + +} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-interpreter/src/test/resources/conf/interpreter.json -- diff --git a/zeppelin-interpreter/src/test/resources/conf/interpreter.json b/zeppelin-interpreter/src/test/resources/conf/interpreter.json new file mode 100644 index 000..45e1d60 --- /dev/null +++ b/zeppelin-interpreter/src/test/resources/conf/interpreter.json @@ -0,0 +1,115 @@ +{ + "interpreterSettings": { +
[5/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
OGGER.warn("Cancel is called when RemoterInterpreter is not opened for " + className); + return; } +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +interpreterProcess.callRemoteFunction(new RemoteInterpreterProcess.RemoteFunction() { + @Override + public Void call(Client client) throws Exception { +client.cancel(sessionId, className, convert(context)); +return null; + } +}); } @Override public FormType getFormType() { -open(); - if (formType != null) { return formType; } -RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); -Client client = null; -try { - client = interpreterProcess.getClient(); -} catch (Exception e1) { - throw new InterpreterException(e1); -} - -boolean broken = false; -try { - formType = FormType.valueOf(client.getFormType(sessionKey, className)); - return formType; -} catch (TException e) { - broken = true; - throw new InterpreterException(e); -} finally { - interpreterProcess.releaseClient(client, broken); +// it is possible to call getFormType before it is opened +synchronized (this) { + if (!isOpened) { +open(); + } } +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +FormType type = interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public FormType call(Client client) throws Exception { +formType = FormType.valueOf(client.getFormType(sessionId, className)); +return formType; + } +}); +return type; } @Override - public int getProgress(InterpreterContext context) { -RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); -if (interpreterProcess == null || !interpreterProcess.isRunning()) { + public int getProgress(final InterpreterContext context) { +if (!isOpened) { + LOGGER.warn("getProgress is called when RemoterInterpreter is not opened for " + className); return 0; } - -Client client = null; -try { - client = interpreterProcess.getClient(); -} catch (Exception e1) { - throw new InterpreterException(e1); -} - -boolean broken = false; -try { - return client.getProgress(sessionKey, className, convert(context)); -} catch (TException e) { - broken = true; - throw new InterpreterException(e); -} finally { - interpreterProcess.releaseClient(client, broken); -} +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public Integer call(Client client) throws Exception { +return client.getProgress(sessionId, className, convert(context)); + } +}); } @Override - public List completion(String buf, int cursor, - InterpreterContext interpreterContext) { -RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); -Client client = null; -try { - client = interpreterProcess.getClient(); -} catch (Exception e1) { - throw new InterpreterException(e1); + public List completion(final String buf, final int cursor, +final InterpreterContext interpreterContext) { +if (!isOpened) { + LOGGER.warn("completion is called when RemoterInterpreter is not opened for " + className); + return new ArrayList<>(); } +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction>() { + @Override + public List call(Client client) throws Exception { +return client.completion(sessionId, className, buf, cursor, +convert(interpreterContext)); + } +}); + } -boolean broken = false; -try { - List completion = client.completion(sessionKey, className, buf, cursor, - convert(interpreterContext)); - return completion; -} catch (TException e) { - broken = true; - throw new InterpreterException(e); -} finally { - interpreterProcess.releaseClient(client, broken); + public String getStatus(final String jobId) { +if (!isOpened) { + LOGGER.warn("getStatus is called when RemoteInterpreter is not opened for " + className); + return Job.Status.UNKNOWN.name(); } +RemoteInterpreterProcess interpreterProcess = getOrCreateInterpreterProcess(); +return interpreterProcess.callRemoteFunction( +new RemoteInterpreterProcess.RemoteFunction() { + @Override + public String call
[1/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
Repository: zeppelin Updated Branches: refs/heads/master 69d58f373 -> d6203c51e http://git-wip-us.apache.org/repos/asf/zeppelin/blob/d6203c51/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 634ac30..f4b8c32 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,31 +17,28 @@ package org.apache.zeppelin.notebook; -import static org.junit.Assert.*; -import static org.mockito.Mockito.mock; - -import com.google.common.collect.Maps; -import java.io.File; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - import com.google.common.collect.Sets; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.AbstractInterpreterTest; +import org.apache.zeppelin.interpreter.ClassloaderInterpreter; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOption; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterResultMessage; +import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.LazyOpenInterpreter; import org.apache.zeppelin.interpreter.mock.MockInterpreter1; import org.apache.zeppelin.interpreter.mock.MockInterpreter2; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; import org.apache.zeppelin.resource.LocalResourcePool; -import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.apache.zeppelin.scheduler.SchedulerFactory; @@ -56,18 +53,35 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.sonatype.aether.RepositoryException; -public class NotebookTest implements JobListenerFactory{ +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class NotebookTest extends AbstractInterpreterTest implements JobListenerFactory { private static final Logger logger = LoggerFactory.getLogger(NotebookTest.class); - private File tmpDir; - private ZeppelinConfiguration conf; private SchedulerFactory schedulerFactory; - private File notebookDir; private Notebook notebook; private NotebookRepo notebookRepo; - private InterpreterFactory factory; - private InterpreterSettingManager interpreterSettingManager; - private DependencyResolver depResolver; private NotebookAuthorization notebookAuthorization; private Credentials credentials; private AuthenticationInfo anonymous = AuthenticationInfo.ANONYMOUS; @@ -75,57 +89,30 @@ public class NotebookTest implements JobListenerFactory{ @Before public void setUp() throws Exception { +System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_PUBLIC.getVarName(), "true"); +System.setProperty(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "mock1,mock2"); +super.setUp(); -tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); -tmpDir.mkdirs(); -new File(tmpDir, "conf").mkdirs(); -notebookDir = new File(tmpDir + "/notebook"); -notebookDir.mkdirs(); - -System.setProperty(ConfVars.ZEPPELIN_CONF_DIR.getVarName(), tmpDir.toString() + "/conf"
[4/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
+job.setStatus(Status.PENDING); + +synchronized (queue) { + queue.add(job); + queue.notify(); +} + } + + public void setMaxConcurrency(int maxConcurrency) { +this.maxConcurrency = maxConcurrency; +synchronized (queue) { + queue.notify(); +} + } + + /** + * Role of the class is get status info from remote process from PENDING to + * RUNNING status. + */ + private class JobStatusPoller extends Thread { +private long initialPeriodMsec; +private long initialPeriodCheckIntervalMsec; +private long checkIntervalMsec; +private volatile boolean terminate; +private JobListener listener; +private Job job; +volatile Status lastStatus; + +public JobStatusPoller(long initialPeriodMsec, +long initialPeriodCheckIntervalMsec, long checkIntervalMsec, Job job, +JobListener listener) { + setName("JobStatusPoller-" + job.getId()); + this.initialPeriodMsec = initialPeriodMsec; + this.initialPeriodCheckIntervalMsec = initialPeriodCheckIntervalMsec; + this.checkIntervalMsec = checkIntervalMsec; + this.job = job; + this.listener = listener; + this.terminate = false; +} + +@Override +public void run() { + long started = System.currentTimeMillis(); + while (terminate == false) { +long current = System.currentTimeMillis(); +long interval; +if (current - started < initialPeriodMsec) { + interval = initialPeriodCheckIntervalMsec; +} else { + interval = checkIntervalMsec; +} + +synchronized (this) { + try { +this.wait(interval); + } catch (InterruptedException e) { +logger.error("Exception in RemoteScheduler while run this.wait", e); + } +} + +if (terminate) { + // terminated by shutdown + break; +} + +Status newStatus = getStatus(); +if (newStatus == Status.UNKNOWN) { // unknown + continue; +} + +if (newStatus != Status.READY && newStatus != Status.PENDING) { + // we don't need more + break; +} + } + terminate = true; +} + +public void shutdown() { + terminate = true; + synchronized (this) { +this.notify(); + } +} + + +private Status getLastStatus() { + if (terminate == true) { +if (job.getErrorMessage() != null) { + return Status.ERROR; +} else if (lastStatus != Status.FINISHED && +lastStatus != Status.ERROR && +lastStatus != Status.ABORT) { + return Status.FINISHED; +} else { + return (lastStatus == null) ? Status.FINISHED : lastStatus; +} + } else { +return (lastStatus == null) ? Status.UNKNOWN : lastStatus; + } +} + +public synchronized Status getStatus() { + if (!remoteInterpreter.isOpened()) { +return getLastStatus(); + } + Status status = Status.valueOf(remoteInterpreter.getStatus(job.getId())); + if (status == Status.UNKNOWN) { +// not found this job in the remote schedulers. +// maybe not submitted, maybe already finished +//Status status = getLastStatus(); +listener.afterStatusChange(job, null, null); +return job.getStatus(); + } + lastStatus = status; + listener.afterStatusChange(job, null, status); + return status; +} + } + + //TODO(zjffdu) need to refactor the schdule module which is too complicated + private class JobRunner implements Runnable, JobListener { +private final Logger logger = LoggerFactory.getLogger(JobRunner.class); +private Scheduler scheduler; +private Job job; +private volatile boolean jobExecuted; +volatile boolean jobSubmittedRemotely; + +public JobRunner(Scheduler scheduler, Job job) { + this.scheduler = scheduler; + this.job = job; + jobExecuted = false; + jobSubmittedRemotely = false; +} + +public boolean isJobSubmittedInRemote() { + return jobSubmittedRemotely; +} + +@Override +public void run() { + if (job.isAborted()) { +synchronized (queue) { + job.setStatus(Status.ABORT); + job.aborted = false; + + running.remove(job); + queue.notify(); +} +jobSubmittedRemotely = true; + +return; + } + + JobStatusPoller jobStatusPoller = new JobStatusPoller(1500, 100, 500, + job, this); + jobStatusPoller.start(); + + if (listener != null) { +listener.jobStarted(scheduler, job); + } + job.run(); + + jobExecuted = true; + jobSubmittedRemotely = true; + + jobStatusPoller.shutdown(); + try { +jobStatusPoller.join(); + } catch (InterruptedException e) { +logger.error("JobStatusPoller interrupted", e); + } + +
[9/9] zeppelin git commit: [ZEPPELIN-2627] Interpreter Component Refactoring
[ZEPPELIN-2627] Interpreter Component Refactoring What is this PR for? I didn't intended to make such large change at the beginning, but found many things are coupled together that I have to make such large change. Several suggestions for you how to review and read it. I move the interpreter package from zeppelin-zengine to zeppelin-interpreter, this is needed for this refactoring. The overall change is the same as I described in the design doc. I would suggest you to read the unit test first. These unit test is very readable and easy to understand what the code is doing now. `InterpreterFactoryTest`, `InterpreterGroupTest`, `InterpreterSettingTest`, `InterpreterSettingManagerTest`, `RemoteInterpreterTest`. Remove the referent counting logic. Now I will kill the interpreter process as long as all the sessions in the same interpreter group is closed. (I plan to add another kind of policy for the interpreter process lifecycle control, ZEPPELIN-2197) The RemoteFunction I introduced is for reducing code duplicates when we use RPC. The changes in `Job.java` and `RemoteScheduler` is for fixing the race issue bug. This bug cause the flaky test we see often in `ZeppelinSparkClusterTest.pySparkTest` What type of PR is it? [Bug Fix | Refactoring] Todos - Task What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-2627 How should this be tested? Unit test is added Screenshots (if appropriate) Questions: Does the licenses files need update? No Is there breaking changes for older versions? No Does this needs documentation? No Author: Jeff Zhang Closes #2554 from zjffdu/ZEPPELIN-2627-2 and squashes the following commits: fa0d435 [Jeff Zhang] minor update 74bcb91 [Jeff Zhang] [ZEPPELIN-2627] Interpreter Component Refactoring Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/d6203c51 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/d6203c51 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/d6203c51 Branch: refs/heads/master Commit: d6203c51ed9eef5e616090326d3dd6dddf21216a Parents: 69d58f3 Author: Jeff Zhang Authored: Fri Sep 1 12:50:46 2017 +0800 Committer: Jeff Zhang Committed: Sun Sep 3 10:41:10 2017 +0800 -- .travis.yml |1 + .../zeppelin/helium/ZeppelinDevServer.java |8 +- .../zeppelin/interpreter/Interpreter.java |1 - .../zeppelin/interpreter/InterpreterGroup.java | 237 +--- .../zeppelin/interpreter/InterpreterRunner.java |9 + .../interpreter/remote/AppendOutputBuffer.java | 54 - .../interpreter/remote/AppendOutputRunner.java | 116 -- .../interpreter/remote/ClientFactory.java | 84 -- .../remote/InterpreterContextRunnerPool.java| 88 -- .../interpreter/remote/RemoteAngularObject.java | 53 - .../remote/RemoteInterpreterEventPoller.java| 571 - .../remote/RemoteInterpreterProcess.java| 242 .../RemoteInterpreterProcessListener.java | 45 - .../remote/RemoteInterpreterServer.java | 64 +- .../zeppelin/resource/ResourcePoolUtils.java| 138 --- .../java/org/apache/zeppelin/scheduler/Job.java | 11 +- .../zeppelin/scheduler/RemoteScheduler.java | 426 --- .../zeppelin/scheduler/SchedulerFactory.java| 63 +- .../zeppelin/tabledata/TableDataProxy.java |1 - .../java/org/apache/zeppelin/util/IdHashes.java | 76 ++ .../java/org/apache/zeppelin/util/Util.java | 76 ++ .../zeppelin/interpreter/DummyInterpreter.java | 43 - .../InterpreterOutputChangeWatcherTest.java | 11 +- .../zeppelin/interpreter/InterpreterTest.java | 38 + .../remote/RemoteInterpreterUtilsTest.java | 33 + .../src/test/resources/conf/interpreter.json| 115 ++ .../interpreter/test/interpreter-setting.json | 42 + .../src/test/resources/log4j.properties |4 +- .../zeppelin/rest/InterpreterRestApi.java |4 +- .../apache/zeppelin/server/ZeppelinServer.java | 37 +- .../apache/zeppelin/socket/NotebookServer.java | 35 +- .../interpreter/mock/MockInterpreter1.java | 75 -- .../zeppelin/rest/AbstractTestRestApi.java |7 +- .../zeppelin/rest/InterpreterRestApiTest.java |6 +- .../zeppelin/socket/NotebookServerTest.java | 13 +- .../src/test/resources/log4j.properties |1 - .../zeppelin/conf/ZeppelinConfiguration.java| 20 +- .../java/org/apache/zeppelin/helium/Helium.java | 54 +- .../helium/HeliumApplicationFactory.java| 127 +- .../interpreter/InterpreterFactory.java | 352 +- .../interpreter/InterpreterGroupFactory.java| 26 - .../interpreter/InterpreterInfoSaving.java | 68 +- .../interpreter/InterpreterSetting.java | 801 ++--- .../interpreter/InterpreterSettingManager.java | 1131 +++-- .../interp
zeppelin git commit: [ZEPPELIN-2876]. Integrate with Livy 0.4.0
Repository: zeppelin Updated Branches: refs/heads/master d6203c51e -> a1c13c6b5 [ZEPPELIN-2876]. Integrate with Livy 0.4.0 ### What is this PR for? This is to integrate livy 0.4.0 which is livy's first apache release. Since the package name is changed, I can not support both livy 0.3.0 and 0.4.0 in the integration test, but since the rest api is not changed. Ideally livy before 0.4.0 should still work. Regarding the integration test, unfortunately I have to comment out some test due to livy's issue, I will enable it after next livy version release. ### What type of PR is it? [Bug Fix | Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2876 ### How should this be tested? integration test is added ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2560 from zjffdu/ZEPPELIN-2876 and squashes the following commits: fdbf39e [Jeff Zhang] [ZEPPELIN-2876]. Integrate with Livy 0.4.0 Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/a1c13c6b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/a1c13c6b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/a1c13c6b Branch: refs/heads/master Commit: a1c13c6b5f13a733aa0a7e20ac03425b1d9be62d Parents: d6203c5 Author: Jeff Zhang Authored: Wed Aug 23 20:34:15 2017 +0800 Committer: Jeff Zhang Committed: Wed Sep 6 11:26:07 2017 +0800 -- .travis.yml | 15 +- livy/pom.xml| 789 +++ .../apache/zeppelin/livy/LivyInterpreterIT.java | 22 +- testing/downloadLivy.sh | 4 +- 4 files changed, 287 insertions(+), 543 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a1c13c6b/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 64ea559..0d9e72a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -109,13 +109,13 @@ matrix: - sudo: required dist: precise jdk: "oraclejdk7" - env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.2.0" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Plivy-0.2 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.1" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-1.6 -Phadoop-2.6 -Pscala-2.10" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" # Test python/pyspark with python 3, livy 0.3 - sudo: required dist: precise jdk: "oraclejdk7" - env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.3.0" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11 -Plivy-0.3" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" + env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.0.0" HADOOP_VER="2.6" LIVY_VER="0.4.0-incubating" PROFILE="-Pspark-2.0 -Phadoop-2.6 -Pscala-2.11" BUILD_FLAG="install -am -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl .,zeppelin-interpreter,zeppelin-display,spark-dependencies,spark,python,livy" TEST_PROJECTS="-Dtest=LivySQLInterpreterTest,org.apache.zeppelin.spark.PySpark*Test,org.apache.zeppelin.python.* -Dpyspark.test.exclude='' -DfailIfNoTests=false" before_install: # check files includ
zeppelin git commit: ZEPPELIN-2645. Adding way to register RemoteInterpreterServer's port into InterpreterProcess
Repository: zeppelin Updated Branches: refs/heads/master b14f5ab95 -> 1812928bf ZEPPELIN-2645. Adding way to register RemoteInterpreterServer's port into InterpreterProcess ### What is this PR for? Rebase PR #2418 , still use thrift as the communication protocol between zeppelin server and interpreter process. We can change it io netty in future when we implement 2 way communication channel between zeppelin server and interpreter process. ### What type of PR is it? [Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2645 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? NO * Does this needs documentation? No Author: Jongyoul Lee Closes #2562 from zjffdu/ZEPPELIN-2645 and squashes the following commits: 82bd8d0 [Jongyoul Lee] ZEPPELIN-2645. Adding way to register RemoteInterpreterServer's port into InterpreterProcess Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1812928b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1812928b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1812928b Branch: refs/heads/master Commit: 1812928bfbecb0cb685fe3233e65ef9d8c84f73f Parents: b14f5ab Author: Jongyoul Lee Authored: Wed Jun 14 19:43:49 2017 +0900 Committer: Jeff Zhang Committed: Thu Sep 7 06:29:55 2017 +0800 -- bin/interpreter.sh | 13 +- conf/zeppelin-site.xml.template | 6 + .../zeppelin/helium/ZeppelinDevServer.java | 4 +- .../remote/RemoteInterpreterServer.java | 148 +++- .../remote/RemoteInterpreterUtils.java | 98 ++- .../interpreter/thrift/CallbackInfo.java| 518 +++ .../RemoteInterpreterCallbackService.java | 879 +++ .../main/thrift/RemoteInterpreterService.thrift | 9 + .../remote/RemoteInterpreterServerTest.java | 6 +- .../remote/RemoteInterpreterUtilsTest.java | 11 + .../zeppelin/conf/ZeppelinConfiguration.java| 8 +- .../interpreter/InterpreterSetting.java | 3 +- .../remote/RemoteInterpreterManagedProcess.java | 129 ++- .../apache/zeppelin/notebook/NotebookTest.java | 6 +- 14 files changed, 1754 insertions(+), 84 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1812928b/bin/interpreter.sh -- diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 1344e31..fd93a06 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -23,7 +23,7 @@ function usage() { echo "usage) $0 -p -d -l -g " } -while getopts "hp:d:l:v:u:g:" o; do +while getopts "hc:p:d:l:v:u:g:" o; do case ${o} in h) usage @@ -32,8 +32,11 @@ while getopts "hp:d:l:v:u:g:" o; do d) INTERPRETER_DIR=${OPTARG} ;; +c) +CALLBACK_HOST=${OPTARG} # This will be used callback host +;; p) -PORT=${OPTARG} +PORT=${OPTARG} # This will be used callback port ;; l) LOCAL_INTERPRETER_REPO=${OPTARG} @@ -202,12 +205,12 @@ fi if [[ -n "${SPARK_SUBMIT}" ]]; then if [[ -n "$ZEPPELIN_IMPERSONATE_USER" ]] && [[ "$ZEPPELIN_IMPERSONATE_SPARK_PROXY_USER" != "false" ]]; then - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} --proxy-user ${ZEPPELIN_IMPERSONATE_USER} ${SPARK_APP_JAR} ${CALLBACK_HOST} ${PORT}` else - INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}\" --driver-java-options \"${JAVA_INTP_OPTS}\" ${SPARK_SUBMIT_OPTIONS} ${ZEPPELIN_SPARK_CONF} ${SPARK_APP_JAR} ${PORT}` + INTERPRETER_RUN_COMMAND+=' '` echo ${SPARK_SUBMIT} --class ${ZEPPELIN_SERVER} --driver-class-path \"${ZEPPELI
zeppelin git commit: ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver
Repository: zeppelin Updated Branches: refs/heads/master 1812928bf -> 1e1b95ea1 ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver ### What is this PR for? Straightforward change for allowing pass non spark.* properties to spark app driver. See discussion here http://apache-zeppelin-users-incubating-mailing-list.75479.x6.nabble.com/Setting-spark-config-properties-in-Zeppelin-0-7-2-td6143.html ### What type of PR is it? [Bug Fix ] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2893 ### How should this be tested? Unit test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2561 from zjffdu/ZEPPELIN-2893 and squashes the following commits: 1342527 [Jeff Zhang] ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/1e1b95ea Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/1e1b95ea Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/1e1b95ea Branch: refs/heads/master Commit: 1e1b95ea1dc5db408773263837c9cbbd6f66b288 Parents: 1812928 Author: Jeff Zhang Authored: Sun Sep 3 10:39:37 2017 +0800 Committer: Jeff Zhang Committed: Thu Sep 7 06:35:31 2017 +0800 -- docs/interpreter/spark.md | 7 + .../apache/zeppelin/spark/SparkInterpreter.java | 27 +++- .../zeppelin/spark/SparkInterpreterTest.java| 9 ++- 3 files changed, 36 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e1b95ea/docs/interpreter/spark.md -- diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 8ba9247..780c60a 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -196,6 +196,13 @@ Staring from 0.6.1 SparkSession is available as variable `spark` when you are us +### How to pass property to SparkConf + +There're 2 kinds of properties that would be passed to SparkConf + + * Standard spark property (prefix with `spark.`). e.g. `spark.executor.memory` will be passed to `SparkConf` + * Non-standard spark property (prefix with `zeppelin.spark.`). e.g. `zeppelin.spark.property_1`, `property_1` will be passed to `SparkConf` + ## Dependency Management There are two ways to load external libraries in Spark interpreter. First is using interpreter setting menu and second is loading Spark properties. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/1e1b95ea/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index df41014..670314b 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -381,9 +381,16 @@ public class SparkInterpreter extends Interpreter { for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); - if (key.startsWith("spark.") && !val.trim().isEmpty()) { -logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); -conf.set(key, val); + if (!val.trim().isEmpty()) { +if (key.startsWith("spark.")) { + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); + conf.set(key, val); +} +if (key.startsWith("zeppelin.spark.")) { + String sparkPropertyKey = key.substring("zeppelin.spark.".length()); + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", sparkPropertyKey, val)); + conf.set(sparkPropertyKey, val); +} } } @@ -509,9 +516,17 @@ public class SparkInterpreter extends Interpreter { for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); - if (key.startsWith("spark.") && !val.trim().isEmpty()) { -logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); -conf.set(key, val); + if (!val.trim().isEmpty()) { +if (key.startsWith("spark.")) { + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); + conf.set(key, val); +
zeppelin git commit: ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 e667bb5a9 -> 4298a426b ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver ### What is this PR for? Straightforward change for allowing pass non spark.* properties to spark app driver. See discussion here http://apache-zeppelin-users-incubating-mailing-list.75479.x6.nabble.com/Setting-spark-config-properties-in-Zeppelin-0-7-2-td6143.html ### What type of PR is it? [Bug Fix ] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2893 ### How should this be tested? Unit test is added ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2561 from zjffdu/ZEPPELIN-2893 and squashes the following commits: 1342527 [Jeff Zhang] ZEPPELIN-2893. Pass non spark.* of spark interpreter properties to spark app driver (cherry picked from commit 1e1b95ea1dc5db408773263837c9cbbd6f66b288) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4298a426 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4298a426 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4298a426 Branch: refs/heads/branch-0.7 Commit: 4298a426b79bee0b25bac3e27631d483f06a172b Parents: e667bb5 Author: Jeff Zhang Authored: Sun Sep 3 10:39:37 2017 +0800 Committer: Jeff Zhang Committed: Thu Sep 7 06:36:12 2017 +0800 -- docs/interpreter/spark.md | 7 + .../apache/zeppelin/spark/SparkInterpreter.java | 27 +++- .../zeppelin/spark/SparkInterpreterTest.java| 9 ++- 3 files changed, 36 insertions(+), 7 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4298a426/docs/interpreter/spark.md -- diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 6d92561..5779177 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -184,6 +184,13 @@ Staring from 0.6.1 SparkSession is available as variable `spark` when you are us +### How to pass property to SparkConf + +There're 2 kinds of properties that would be passed to SparkConf + + * Standard spark property (prefix with `spark.`). e.g. `spark.executor.memory` will be passed to `SparkConf` + * Non-standard spark property (prefix with `zeppelin.spark.`). e.g. `zeppelin.spark.property_1`, `property_1` will be passed to `SparkConf` + ## Dependency Management There are two ways to load external libraries in Spark interpreter. First is using interpreter setting menu and second is loading Spark properties. http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4298a426/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index e7bf265..5b6dc76 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -343,9 +343,16 @@ public class SparkInterpreter extends Interpreter { for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); - if (key.startsWith("spark.") && !val.trim().isEmpty()) { -logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); -conf.set(key, val); + if (!val.trim().isEmpty()) { +if (key.startsWith("spark.")) { + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); + conf.set(key, val); +} +if (key.startsWith("zeppelin.spark.")) { + String sparkPropertyKey = key.substring("zeppelin.spark.".length()); + logger.debug(String.format("SparkConf: key = [%s], value = [%s]", sparkPropertyKey, val)); + conf.set(sparkPropertyKey, val); +} } } @@ -474,9 +481,17 @@ public class SparkInterpreter extends Interpreter { for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); - if (key.startsWith("spark.") && !val.trim().isEmpty()) { -logger.debug(String.format("SparkConf: key = [%s], value = [%s]", key, val)); -conf.set(key, val); + if (!val.trim().isEmpty()) { +if (key.startsWith("spark.")) { + logger.debug(String.format("SparkConf
zeppelin git commit: ZEPPELIN-2377. Hive Support can not be enabled in spark master
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 05adf7ee2 -> e51988a94 ZEPPELIN-2377. Hive Support can not be enabled in spark master ### What is this PR for? The root cause is that `org.apache.spark.sql.hive.HiveSessionState` is removed in spark master. I change it to `org.apache.spark.sql.hive.execution.InsertIntoHiveTable` which is existed early in spark 1.0. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2377 ### How should this be tested? Verify it manually in spark master, spark 2.1.0 and spark 1.6.2 ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2237 from zjffdu/ZEPPELIN-2377 and squashes the following commits: d16196b [Jeff Zhang] ZEPPELIN-2377. Hive Support can not be enabled in spark master (cherry picked from commit 74c0408d30927b11c99a8e134284be4156b19f8a) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/e51988a9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/e51988a9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/e51988a9 Branch: refs/heads/branch-0.7 Commit: e51988a94fdcb63c8143c2dc5fdb69dba050b3d6 Parents: 05adf7e Author: Jeff Zhang Authored: Sun Apr 9 15:46:32 2017 +0800 Committer: Jeff Zhang Committed: Fri Sep 8 12:00:41 2017 +0800 -- .../src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/e51988a9/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 5b6dc76..23a698a 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -215,7 +215,7 @@ public class SparkInterpreter extends Interpreter { */ private boolean hiveClassesArePresent() { try { - this.getClass().forName("org.apache.spark.sql.hive.HiveSessionState"); + this.getClass().forName("org.apache.spark.sql.hive.execution.InsertIntoHiveTable"); this.getClass().forName("org.apache.hadoop.hive.conf.HiveConf"); return true; } catch (ClassNotFoundException | NoClassDefFoundError e) {
zeppelin git commit: ZEPPELIN-2933. Code Refactoring of ZEPPELIN-1515 follow up
Repository: zeppelin Updated Branches: refs/heads/master 3fb67f9cc -> 3f591c232 ZEPPELIN-2933. Code Refactoring of ZEPPELIN-1515 follow up ### What is this PR for? This is a refactoring PR of ZEPPELIN-1515. Because hadoop's FileSystem API not only works with hdfs, but also other hadoop compatible filesystem. So in this PR I rename it to `FileSystemNotebookRepo` ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2933 ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2588 from zjffdu/ZEPPELIN-2933 and squashes the following commits: 45d1e9b [Jeff Zhang] ZEPPELIN-2993. Code Refactoring of ZEPPELIN-1515 follow up Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/3f591c23 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/3f591c23 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/3f591c23 Branch: refs/heads/master Commit: 3f591c2327532679159019b9d7486805ef6d0768 Parents: 3fb67f9 Author: Jeff Zhang Authored: Thu Sep 14 13:23:33 2017 +0800 Committer: Jeff Zhang Committed: Fri Sep 15 09:03:14 2017 +0800 -- conf/zeppelin-site.xml.template | 6 +- docs/setup/storage/storage.md | 10 +- .../remote/RemoteInterpreterServerTest.java | 2 +- .../notebook/repo/FileSystemNotebookRepo.java | 206 +++ .../notebook/repo/HdfsNotebookRepo.java | 200 -- .../zeppelin/notebook/repo/VFSNotebookRepo.java | 1 + .../repo/FileSystemNotebookRepoTest.java| 101 + .../notebook/repo/HdfsNotebookRepoTest.java | 101 - 8 files changed, 317 insertions(+), 310 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/3f591c23/conf/zeppelin-site.xml.template -- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index ce3ffaa..f1bfb61 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -173,11 +173,11 @@ --> -
zeppelin git commit: [ZEPPELIN-2928] Hotfix on maven set version functionality
Repository: zeppelin Updated Branches: refs/heads/master 3f591c232 -> 9c1fca182 [ZEPPELIN-2928] Hotfix on maven set version functionality ### What is this PR for? Hotfix on a recent commit that removed the version soft-coded variable and introduced hard-coded references to version tag. ### What type of PR is it? [Bug Fix | Hot Fix ] ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2928 ### How should this be tested? Run "mvn versions:set -DprocessDependencies=false -DnewVersion=0.8.0-SNAPSHOT-123" ### Questions: * Does the licenses files need update? N * Is there breaking changes for older versions? N * Does this needs documentation? N Author: Nelson Costa Closes #2584 from necosta/zeppelin-2928 and squashes the following commits: 5fd2550 [Nelson Costa] [ZEPPELIN-2928] Hotfix on maven set version functionality f5ccf89 [Nelson Costa] [ZEPPELIN-2928] Hotfix on maven set version functionality Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9c1fca18 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9c1fca18 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9c1fca18 Branch: refs/heads/master Commit: 9c1fca182314c817e03250f0760aff2b0158cc7a Parents: 3f591c2 Author: Nelson Costa Authored: Thu Sep 14 09:43:52 2017 +0100 Committer: Jeff Zhang Committed: Fri Sep 15 14:08:17 2017 +0800 -- zeppelin-distribution/pom.xml | 4 ++-- zeppelin-jupyter/pom.xml | 2 +- zeppelin-server/pom.xml | 2 +- zeppelin-zengine/pom.xml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9c1fca18/zeppelin-distribution/pom.xml -- diff --git a/zeppelin-distribution/pom.xml b/zeppelin-distribution/pom.xml index 36c3522..ed05c93 100644 --- a/zeppelin-distribution/pom.xml +++ b/zeppelin-distribution/pom.xml @@ -75,13 +75,13 @@ - zeppelin-server ${project.groupId} + zeppelin-server ${project.version} - zeppelin-web ${project.groupId} + zeppelin-web ${project.version} war http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9c1fca18/zeppelin-jupyter/pom.xml -- diff --git a/zeppelin-jupyter/pom.xml b/zeppelin-jupyter/pom.xml index 914ec51..c6c688a 100644 --- a/zeppelin-jupyter/pom.xml +++ b/zeppelin-jupyter/pom.xml @@ -63,7 +63,7 @@ org.apache.zeppelin zeppelin-markdown - ${zeppelin.version} + ${project.version} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9c1fca18/zeppelin-server/pom.xml -- diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index e69fba4..24d5ee7 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -87,7 +87,7 @@ ${project.groupId} zeppelin-zengine - 0.8.0-SNAPSHOT + ${project.version} com.fasterxml.jackson.core http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9c1fca18/zeppelin-zengine/pom.xml -- diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index b3d5c63..314ca18 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -57,7 +57,7 @@ ${project.groupId} zeppelin-interpreter - 0.8.0-SNAPSHOT + ${project.version}
[1/2] zeppelin git commit: ZEPPELIN-2898. Support Yarn-Cluster for Spark Interpreter
Repository: zeppelin Updated Branches: refs/heads/master d25639caa -> 5d7151097 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5d715109/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 98a08d9..d62c308 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -73,9 +73,7 @@ public class VFSNotebookRepoTest extends AbstractInterpreterTest implements JobL @After public void tearDown() throws Exception { -if (!FileUtils.deleteQuietly(testRootDir)) { - LOG.error("Failed to delete {} ", testRootDir.getName()); -} +super.tearDown(); } @Test
[2/2] zeppelin git commit: ZEPPELIN-2898. Support Yarn-Cluster for Spark Interpreter
ZEPPELIN-2898. Support Yarn-Cluster for Spark Interpreter ### What is this PR for? This is the first version for supporting yarn-cluster of `SparkInterpreter`. I just delegate all the function to `spark-submit` as yarn-cluster is natively supported by spark, we don't need to reinvent the wheel. But there's still improvement to be done in future, e.g. I put some spark specific logic in `InterpreterSetting` which is not a good practise. I plan to improve it when I refactor the `Interpreter` class (ZEPPELIN-2685). Besides that, I also add `MiniHadoopCluster` & `MiniZeppelin` which help for the integration test of yarn-client & yarn-cluster mode, otherwise I have to manually verify yarn-client & yarn-cluster mode which would easily cause regression issue in future. To be noticed: * SPARK_HOME must be specified for yarn-cluster mode * HADOOP_CONF_DIR must be specified for yarn-cluster mode ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? https://github.com/zjffdu/zeppelin/tree/ZEPPELIN-2898 ### How should this be tested? System test is added in `SparkInterpreterIT`. ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2577 from zjffdu/ZEPPELIN-2898 and squashes the following commits: 9da7c4b [Jeff Zhang] ZEPPELIN-2898. Support Yarn-Cluster for Spark Interpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/5d715109 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/5d715109 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/5d715109 Branch: refs/heads/master Commit: 5d7151097e171c5ec9f22f150ac4ce92b5512013 Parents: d25639c Author: Jeff Zhang Authored: Mon Sep 4 21:54:56 2017 +0800 Committer: Jeff Zhang Committed: Tue Sep 19 06:40:51 2017 +0800 -- .travis.yml | 4 +- bin/common.sh | 6 +- bin/interpreter.sh | 12 +- conf/log4j_yarn_cluster.properties | 23 ++ docs/interpreter/spark.md | 2 +- .../zeppelin/python/IPythonInterpreter.java | 44 +-- .../zeppelin/spark/IPySparkInterpreter.java | 11 +- .../zeppelin/spark/PySparkInterpreter.java | 22 +- .../org/apache/zeppelin/spark/PythonUtils.java | 96 +++ .../apache/zeppelin/spark/SparkInterpreter.java | 99 +-- .../src/main/resources/interpreter-setting.json | 4 +- .../zeppelin/spark/IPySparkInterpreterTest.java | 1 + .../spark/PySparkInterpreterMatplotlibTest.java | 2 +- .../zeppelin/spark/PySparkInterpreterTest.java | 2 +- .../interpreter/InterpreterContext.java | 47 ++- zeppelin-server/pom.xml | 12 + .../zeppelin/integration/AuthenticationIT.java | 2 +- .../integration/InterpreterModeActionsIT.java | 2 +- .../integration/PersonalizeActionsIT.java | 2 +- .../zeppelin/rest/AbstractTestRestApi.java | 8 +- zeppelin-zengine/pom.xml| 286 ++- .../zeppelin/conf/ZeppelinConfiguration.java| 2 +- .../interpreter/InterpreterSetting.java | 131 - .../interpreter/InterpreterSettingManager.java | 15 +- .../remote/RemoteInterpreterEventPoller.java| 6 +- .../remote/RemoteInterpreterManagedProcess.java | 2 + .../remote/RemoteInterpreterProcess.java| 2 +- .../interpreter/AbstractInterpreterTest.java| 23 +- .../zeppelin/interpreter/MiniHadoopCluster.java | 114 .../zeppelin/interpreter/MiniZeppelin.java | 68 + .../interpreter/SparkInterpreterModeTest.java | 147 ++ .../notebook/repo/VFSNotebookRepoTest.java | 4 +- 32 files changed, 1007 insertions(+), 194 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/5d715109/.travis.yml -- diff --git a/.travis.yml b/.travis.yml index 0d9e72a..4495aa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,7 +66,8 @@ matrix: # Several tests were excluded from this configuration due to the following issues: # HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470 # After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property -- jdk: "oraclejdk8" +- sudo: required + jdk: "oraclejdk8" dist: precise env: PYTHON="3" SCALA_VER="2.11" SPARK_VER="2.2.0" HADOOP_VER="2.6" PROFILE="-Pspark-2.2 -Pweb-ci -Pscalding -Phelium-dev -Pexamples -Pscala-2.11" BUILD_FL
zeppelin git commit: ZEPPELIN-2943 Streaming output will be closed in 30 seconds for IPythonInterpreter
Repository: zeppelin Updated Branches: refs/heads/master 5d7151097 -> 7f16fc217 ZEPPELIN-2943 Streaming output will be closed in 30 seconds for IPythonInterpreter ### What is this PR for? Straightforward fix to set the timeout as 100 years so that spark streaming app won't be stopped unless 100 years later. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2943 ### How should this be tested? Tested manually, see screenshot. ### Screenshots (if appropriate) ![streaming](https://user-images.githubusercontent.com/164491/30589783-789a99c6-9d01-11e7-88a3-c36d49321541.gif) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2593 from zjffdu/ZEPPELIN-2943 and squashes the following commits: 9144409 [Jeff Zhang] Fix a test bug 0923b3c [Jeff Zhang] ZEPPELIN-2943 Streaming output will be closed in 30 seconds for IPythonInterpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/7f16fc21 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/7f16fc21 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/7f16fc21 Branch: refs/heads/master Commit: 7f16fc217732b486f5df041f0cfa398d5650a500 Parents: 5d71510 Author: Jeff Zhang Authored: Wed Sep 20 06:55:01 2017 +0800 Committer: Jeff Zhang Committed: Wed Sep 20 08:12:07 2017 +0800 -- python/src/main/resources/grpc/python/ipython_server.py | 5 +++-- .../apache/zeppelin/interpreter/AbstractInterpreterTest.java| 1 + 2 files changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/7f16fc21/python/src/main/resources/grpc/python/ipython_server.py -- diff --git a/python/src/main/resources/grpc/python/ipython_server.py b/python/src/main/resources/grpc/python/ipython_server.py index 1d92766..98fa616 100644 --- a/python/src/main/resources/grpc/python/ipython_server.py +++ b/python/src/main/resources/grpc/python/ipython_server.py @@ -35,7 +35,7 @@ else: import queue as queue -TIMEOUT = 30 +TIMEOUT = 60*60*24*365*100 # 100 years class IPython(ipython_pb2_grpc.IPythonServicer): @@ -50,7 +50,8 @@ class IPython(ipython_pb2_grpc.IPythonServicer): self._status = ipython_pb2.RUNNING def execute(self, request, context): -print("execute code: " + request.code) +print("execute code:\n") +print(request.code) sys.stdout.flush() stdout_queue = queue.Queue(maxsize = 10) stderr_queue = queue.Queue(maxsize = 10) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/7f16fc21/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java index 9ab2137..ad3dd99 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/AbstractInterpreterTest.java @@ -56,6 +56,7 @@ public abstract class AbstractInterpreterTest { System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), notebookDir.getAbsolutePath()); conf = new ZeppelinConfiguration(); + conf.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER.getVarName(), "test,mock1,mock2,mock_resource_pool"); interpreterSettingManager = new InterpreterSettingManager(conf, mock(AngularObjectRegistryListener.class), mock(RemoteInterpreterProcessListener.class), mock(ApplicationEventListener.class)); interpreterFactory = new InterpreterFactory(interpreterSettingManager);
zeppelin git commit: [ZEPPELIN-2641] Change encoding to UTF-8 when sending request to Livy
Repository: zeppelin Updated Branches: refs/heads/master 0d13c0b56 -> f7c47af9b [ZEPPELIN-2641] Change encoding to UTF-8 when sending request to Livy ### What is this PR for? Change encoding of the request sent from Zeppelin to Livy to UTF-8. In this way, Zeppelin can support many more language than using ISO-8895-1 by default. ### What type of PR is it? Bug Fix ### What is the Jira issue? [ZEPPELIN-2641](https://issues.apache.org/jira/browse/ZEPPELIN-2641) ### How should this be tested? Build from source. Open a Livy note book. Run some simple print command with Chinese or Korean, see whether the return can show the character correctly ### Screenshots (if appropriate) before ![image](https://user-images.githubusercontent.com/14201792/27174528-11d45216-51ef-11e7-8f46-2f2e8347a3de.png) after ![image](https://user-images.githubusercontent.com/14201792/27174517-08cdba04-51ef-11e7-989c-88e516b2d265.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: æ±ªèµ«æ¬ Closes #2412 from qwemicheal/utf-8 and squashes the following commits: a72ffeb [汪赫æ¬] change utf8Str to explicit Chinese 8f144bc [汪赫æ¬] add unit test f689a7e [汪赫æ¬] add utf-8 encoding Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/f7c47af9 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/f7c47af9 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/f7c47af9 Branch: refs/heads/master Commit: f7c47af9b552843f63935a59302de31519ae6b97 Parents: 0d13c0b Author: æ±ªèµ«æ¬ Authored: Mon Jun 19 12:39:55 2017 +0800 Committer: Jeff Zhang Committed: Thu Sep 28 13:03:11 2017 +0800 -- .../org/apache/zeppelin/livy/BaseLivyInterpreter.java| 3 ++- .../java/org/apache/zeppelin/livy/LivyInterpreterIT.java | 11 +++ 2 files changed, 13 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7c47af9/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index a5c87f8..ccab09b 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -41,6 +41,7 @@ import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; +import org.springframework.http.MediaType; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; @@ -520,7 +521,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { targetURL = livyURL + targetURL; LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData); HttpHeaders headers = new HttpHeaders(); -headers.add("Content-Type", "application/json"); +headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE); headers.add("X-Requested-By", "zeppelin"); ResponseEntity response = null; try { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/f7c47af9/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java -- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 60c9043..9a0aef4 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -28,6 +28,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Properties; @@ -555,6 +556,16 @@ public class LivyInterpreterIT { assertTrue(result.message().get(0).getData().contains("Traceback")); } +// test utf-8 Encoding +try { + String utf8Str = "ä½ ä½ ä½ ä½ ä½ ä½ å¥½"; + InterpreterResult result = pysparkInterpreter.interpret("print(\""+utf8Str+"\")", context); + assertEquals(InterpreterResult.Code.SUCCESS, result.code()); + assertTrue(result.message().get(0).getData().contains(utf8Str)); +}catch (Exception e) { + e.printStackTrace(); +} + try { InterpreterResult result = pysparkInterpreter.interpret("sc.version", context); assertEquals(InterpreterResult.Code.SUCCESS, result.code());
zeppelin git commit: [ZEPPELIN-2641] Change encoding to UTF-8 when sending request to Livy
Repository: zeppelin Updated Branches: refs/heads/branch-0.7 8a2b46a25 -> 75df4aacc [ZEPPELIN-2641] Change encoding to UTF-8 when sending request to Livy Change encoding of the request sent from Zeppelin to Livy to UTF-8. In this way, Zeppelin can support many more language than using ISO-8895-1 by default. Bug Fix [ZEPPELIN-2641](https://issues.apache.org/jira/browse/ZEPPELIN-2641) Build from source. Open a Livy note book. Run some simple print command with Chinese or Korean, see whether the return can show the character correctly before ![image](https://user-images.githubusercontent.com/14201792/27174528-11d45216-51ef-11e7-8f46-2f2e8347a3de.png) after ![image](https://user-images.githubusercontent.com/14201792/27174517-08cdba04-51ef-11e7-989c-88e516b2d265.png) * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: æ±ªèµ«æ¬ Closes #2412 from qwemicheal/utf-8 and squashes the following commits: a72ffeb [汪赫æ¬] change utf8Str to explicit Chinese 8f144bc [汪赫æ¬] add unit test f689a7e [汪赫æ¬] add utf-8 encoding (cherry picked from commit f7c47af9b552843f63935a59302de31519ae6b97) Signed-off-by: Jeff Zhang Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/75df4aac Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/75df4aac Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/75df4aac Branch: refs/heads/branch-0.7 Commit: 75df4aacca0907d251b06da87d9a49920683e48b Parents: 8a2b46a Author: æ±ªèµ«æ¬ Authored: Mon Jun 19 12:39:55 2017 +0800 Committer: Jeff Zhang Committed: Thu Sep 28 13:04:43 2017 +0800 -- .../main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java | 3 ++- .../src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/75df4aac/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index 022aef8..2c92bbf 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -41,6 +41,7 @@ import org.apache.zeppelin.interpreter.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; +import org.springframework.http.MediaType; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.http.ResponseEntity; @@ -492,7 +493,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { targetURL = livyURL + targetURL; LOGGER.debug("Call rest api in {}, method: {}, jsonData: {}", targetURL, method, jsonData); HttpHeaders headers = new HttpHeaders(); -headers.add("Content-Type", "application/json"); +headers.add("Content-Type", MediaType.APPLICATION_JSON_UTF8_VALUE); headers.add("X-Requested-By", "zeppelin"); ResponseEntity response = null; try { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/75df4aac/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java -- diff --git a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java index 302f002..82fd806 100644 --- a/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java +++ b/livy/src/test/java/org/apache/zeppelin/livy/LivyInterpreterIT.java @@ -29,6 +29,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Properties;
zeppelin git commit: ZEPPELIN-2942. Remove non-remote interpreter approach
Repository: zeppelin Updated Branches: refs/heads/master e28712c48 -> 0c64d9ca6 ZEPPELIN-2942. Remove non-remote interpreter approach ### What is this PR for? This PR is just trying to remove non-remote interpreter approach. ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2942 ### How should this be tested? Current test is passed ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2597 from zjffdu/ZEPPELIN-2942 and squashes the following commits: 42fe419 [Jeff Zhang] ZEPPELIN-2942. Remove non-remote interpreter approach Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/0c64d9ca Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/0c64d9ca Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/0c64d9ca Branch: refs/heads/master Commit: 0c64d9ca676e48a749db9879fa3cebc06eb78b54 Parents: e28712c Author: Jeff Zhang Authored: Fri Sep 22 11:28:42 2017 +0800 Committer: Jeff Zhang Committed: Tue Oct 10 06:32:36 2017 +0800 -- .../zeppelin/interpreter/InterpreterOption.java | 19 + .../interpreter/InterpreterInfoSaving.java | 5 -- .../interpreter/InterpreterSetting.java | 84 ++-- .../interpreter/InterpreterSettingManager.java | 2 +- .../interpreter/InterpreterSettingTest.java | 9 --- .../ManagedInterpreterGroupTest.java| 1 - .../remote/RemoteAngularObjectTest.java | 1 - .../RemoteInterpreterOutputTestStream.java | 1 - .../remote/RemoteInterpreterTest.java | 1 - .../zeppelin/scheduler/RemoteSchedulerTest.java | 1 - 10 files changed, 10 insertions(+), 114 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0c64d9ca/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java -- diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java index 37a0d99..e8a9225 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/interpreter/InterpreterOption.java @@ -28,7 +28,8 @@ public class InterpreterOption { public static final transient String SCOPED = "scoped"; public static final transient String ISOLATED = "isolated"; - boolean remote; + // always set it as true, keep this field just for backward compatibility + boolean remote = true; String host = null; int port = -1; @@ -77,14 +78,9 @@ public class InterpreterOption { } public InterpreterOption() { -this(false); } - public InterpreterOption(boolean remote) { -this(remote, SHARED, SHARED); - } - - public InterpreterOption(boolean remote, String perUser, String perNote) { + public InterpreterOption(String perUser, String perNote) { if (perUser == null) { throw new NullPointerException("perUser can not be null."); } @@ -92,7 +88,6 @@ public class InterpreterOption { throw new NullPointerException("perNote can not be null."); } -this.remote = remote; this.perUser = perUser; this.perNote = perNote; } @@ -112,14 +107,6 @@ public class InterpreterOption { return option; } - public boolean isRemote() { -return remote; - } - - public void setRemote(boolean remote) { -this.remote = remote; - } - public String getHost() { return host; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/0c64d9ca/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index d7593d5..3d9c2c3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -63,11 +63,6 @@ public class InterpreterInfoSaving implements JsonSerializable { if (infoSaving != null && infoSaving.interpreterSettings != null) { for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { - // Always use separate interpreter process - // While we decide
zeppelin git commit: [MINOR][Code Refactoring] Configuration name change for keytab and principal used for FileSystemNotebookRepo
Repository: zeppelin Updated Branches: refs/heads/master fc9d542b7 -> a424f5c65 [MINOR][Code Refactoring] Configuration name change for keytab and principal used for FileSystemNotebookRepo ### What is this PR for? Trivial change for configuration name change. ### What type of PR is it? [Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * No jira created ### How should this be tested? * First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2620 from zjffdu/minor_refactoring and squashes the following commits: c68ce76 [Jeff Zhang] [MINOR][Code Refactoring] Configuration name change for keytab and principal used for FileSystemNotebookRepo Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/a424f5c6 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/a424f5c6 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/a424f5c6 Branch: refs/heads/master Commit: a424f5c655623c8128824f95a673ad7cb06d09a7 Parents: fc9d542 Author: Jeff Zhang Authored: Wed Oct 11 15:54:59 2017 +0800 Committer: Jeff Zhang Committed: Wed Oct 11 20:41:21 2017 +0800 -- conf/zeppelin-site.xml.template | 4 ++-- docs/setup/storage/storage.md | 2 +- .../java/org/apache/zeppelin/conf/ZeppelinConfiguration.java | 4 ++-- .../apache/zeppelin/notebook/repo/FileSystemNotebookRepo.java | 7 --- 4 files changed, 9 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a424f5c6/conf/zeppelin-site.xml.template -- diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index f1bfb61..b25ba19 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -181,13 +181,13 @@ - zeppelin.hdfs.keytab + zeppelin.server.kerberos.keytab keytab for accessing kerberized hdfs - zeppelin.hdfs.principal + zeppelin.server.kerberos.principal principal for accessing kerberized hdfs http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a424f5c6/docs/setup/storage/storage.md -- diff --git a/docs/setup/storage/storage.md b/docs/setup/storage/storage.md index 0b65f91..f6b8b5c 100644 --- a/docs/setup/storage/storage.md +++ b/docs/setup/storage/storage.md @@ -57,7 +57,7 @@ To enable versioning for all your local notebooks though a standard Git reposito ## Notebook Storage in hadoop compatible file system repository Notes may be stored in hadoop compatible file system such as hdfs, so that multiple Zeppelin instances can share the same notes. It supports all the versions of hadoop 2.x. If you use `FileSystemNotebookRepo`, then `zeppelin.notebook.dir` is the path on the hadoop compatible file system. And you need to specify `HADOOP_CONF_DIR` in `zeppelin-env.sh` so that zeppelin can find the right hadoop configuration files. -If your hadoop cluster is kerberized, then you need to specify `zeppelin.hdfs.keytab` and `zeppelin.hdfs.principal` +If your hadoop cluster is kerberized, then you need to specify `zeppelin.server.kerberos.keytab` and `zeppelin.server.kerberos.principal` ``` http://git-wip-us.apache.org/repos/asf/zeppelin/blob/a424f5c6/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index f329d47..3a82bc5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -693,8 +693,8 @@ public class ZeppelinConfiguration extends XMLConfiguration { ZEPPELIN_SERVER_STRICT_TRANSPORT("zeppelin.server.strict.transport", "max-age=631138519"), ZEPPELIN_SERVER_X_XSS_PROTECTION("zeppelin.server.xxss.protection", "1"), -ZEPPELIN_HDFS_KEYTAB("zeppelin.hdfs.keytab", ""), -ZEPPELIN_HDFS_PRINCIPAL("zeppelin.hdfs.principal", ""), +ZEPPELIN_SERVER_KERBEROS_KEYTAB("zeppelin.server.kerberos.k
zeppelin git commit: ZEPPELIN-2953 Allow custom http header for livy interpreter
Repository: zeppelin Updated Branches: refs/heads/master a424f5c65 -> 862871f67 ZEPPELIN-2953 Allow custom http header for livy interpreter ### What is this PR for? This PR is trying to allow user to add custom http headers when calling livy rest api. User just need to specify `zeppelin.livy.http.headers` in livy interpreter setting ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2953 ### How should this be tested? Outline the steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2600 from zjffdu/ZEPPELIN-2953 and squashes the following commits: 720d8d0 [Jeff Zhang] ZEPPELIN-2953 Allow custom http header for livy interpreter Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/862871f6 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/862871f6 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/862871f6 Branch: refs/heads/master Commit: 862871f67afdc3a6357a759e1543f821bed7d7d2 Parents: a424f5c Author: Jeff Zhang Authored: Tue Sep 26 10:12:48 2017 +0800 Committer: Jeff Zhang Committed: Fri Oct 13 09:28:10 2017 +0800 -- docs/interpreter/livy.md| 7 - .../zeppelin/livy/BaseLivyInterpreter.java | 33 .../zeppelin/livy/LivySQLInterpreterTest.java | 8 + 3 files changed, 47 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/862871f6/docs/interpreter/livy.md -- diff --git a/docs/interpreter/livy.md b/docs/interpreter/livy.md index 1741a80..09bf6e1 100644 --- a/docs/interpreter/livy.md +++ b/docs/interpreter/livy.md @@ -144,7 +144,12 @@ Example: `spark.driver.memory` to `livy.spark.driver.memory` zeppelin.livy.ssl.trustStorePassword password for trustStore file. Used when livy ssl is enabled - + + +zeppelin.livy.http.headers +key_1: value_1; key_2: value_2 +custom http headers when calling livy rest api. Each http header is separated by `;`, and each header is one key value pair where key value is separated by `:` + **We remove livy.spark.master in zeppelin-0.7. Because we sugguest user to use livy 0.3 in zeppelin-0.7. And livy 0.3 don't allow to specify livy.spark.master, it enfornce yarn-cluster mode.** http://git-wip-us.apache.org/repos/asf/zeppelin/blob/862871f6/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java -- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java index ccab09b..b725348 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/BaseLivyInterpreter.java @@ -62,6 +62,8 @@ import java.util.Map; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** @@ -80,6 +82,7 @@ public abstract class BaseLivyInterpreter extends Interpreter { protected boolean displayAppInfo; protected LivyVersion livyVersion; private RestTemplate restTemplate; + private Map customHeaders = new HashMap<>(); Set paragraphsToCancel = Collections.newSetFromMap( new ConcurrentHashMap()); @@ -96,6 +99,33 @@ public abstract class BaseLivyInterpreter extends Interpreter { this.pullStatusInterval = Integer.parseInt( property.getProperty("zeppelin.livy.pull_status.interval.millis", 1000 + "")); this.restTemplate = createRestTemplate(); +if (!StringUtils.isBlank(property.getProperty("zeppelin.livy.http.headers"))) { + String[] headers = property.getProperty("zeppelin.livy.http.headers").split(";"); + for (String header : headers) { +String[] splits = header.split(":", -1); +if (splits.length != 2) { + throw new RuntimeException("Invalid format of http headers: " + header + + ", valid http header format is HEADER_NAME:HEADER_VALUE"); +} +customHeaders.put(splits[0].trim(), envSubstitute(splits[1].trim())); + } +} + } + + private String envSubstitute(String value) { +String newValue = new String(value); +Pattern pattern = Pattern.compile("\\$\\{(.*)\\}"); +Matcher matcher = pattern.matcher(value); +whil
[1/2] zeppelin git commit: ZEPPELIN-2982. Copy interpreter-setting.json to interpreter dir
Repository: zeppelin Updated Branches: refs/heads/master 862871f67 -> 4a3057fa8 http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4a3057fa/shell/pom.xml -- diff --git a/shell/pom.xml b/shell/pom.xml index 58d8900..0157d86 100644 --- a/shell/pom.xml +++ b/shell/pom.xml @@ -20,10 +20,10 @@ 4.0.0 -zeppelin +interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT -.. +../interpreter-parent org.apache.zeppelin @@ -33,17 +33,18 @@ Zeppelin: Shell interpreter +sh + 3.4 1.3 +sh ${project.groupId} zeppelin-interpreter - ${project.version} - provided @@ -79,54 +80,12 @@ maven-enforcer-plugin - - -enforce -none - - - maven-dependency-plugin - - -copy-dependencies -package - - copy-dependencies - - - ${project.build.directory}/../../interpreter/sh - false - false - true - runtime - - - -copy-artifact -package - - copy - - - ${project.build.directory}/../../interpreter/sh - false - false - true - runtime - - - ${project.groupId} - ${project.artifactId} - ${project.version} - ${project.packaging} - - - - - + + +maven-resources-plugin http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4a3057fa/zeppelin-interpreter/pom.xml -- diff --git a/zeppelin-interpreter/pom.xml b/zeppelin-interpreter/pom.xml index d08ce4e..00184da 100644 --- a/zeppelin-interpreter/pom.xml +++ b/zeppelin-interpreter/pom.xml @@ -232,4 +232,79 @@ test + + + + + + + maven-enforcer-plugin + + + enforce + none + + + + + + maven-dependency-plugin + + + copy-dependencies + package + +copy-dependencies + + + ${basedir}/interpreter/${project.name} +false +false +true +runtime + + + + copy-artifact + package + +copy + + + ${basedir}/interpreter/${project.name} +false +false +true + + +${project.groupId} +${project.artifactId} +${project.version} +${project.packaging} + + + + + + + + + maven-resources-plugin + + + copy-resources + package + +resources + + + ${basedir}/interpreter/{project.name} + + + + + + + + http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4a3057fa/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 4b130e3..9dfce21 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -192,8 +192,15 @@ public class InterpreterSettingManager { Map mergedProperties = new HashMap<>(InterpreterSetting.convertInterpreterProperties( interpreterSettingTemplate.getProperties())); - mergedProperties.putAll(InterpreterSetting.convertInterpreterProperties( - savedInterpreterSetting.getProperties())); + Map savedProperties = InterpreterSetting + .convertInterpreterProperties(savedInterpreterSetting.getProperties()); + for (Map.Entry entry : savedProperties.entrySet()) { +// only merge prope
[2/2] zeppelin git commit: ZEPPELIN-2982. Copy interpreter-setting.json to interpreter dir
ZEPPELIN-2982. Copy interpreter-setting.json to interpreter dir ### What is this PR for? 2 main changes. * Introduce `interpreter-parent` module, so that all the interpreter can reuse the plugin defined in `interpreter-parent` * Add new plugin for `copying interpreter-setting.json` to interpreter dir ### What type of PR is it? [ Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2982 ### How should this be tested? * First time? Setup Travis CI as described on https://zeppelin.apache.org/contribution/contributions.html#continuous-integration * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2613 from zjffdu/ZEPPELIN-2982 and squashes the following commits: d32 [Jeff Zhang] update doc 8f643a7 [Jeff Zhang] ZEPPELIN-2982. Copy interpreter-setting.json to interpreter dir Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/4a3057fa Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/4a3057fa Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/4a3057fa Branch: refs/heads/master Commit: 4a3057fa87fbe092f4a6b8dcc34e8f6866b7c700 Parents: 862871f Author: Jeff Zhang Authored: Thu Oct 12 09:29:25 2017 +0800 Committer: Jeff Zhang Committed: Fri Oct 13 13:51:22 2017 +0800 -- alluxio/pom.xml | 53 +-- angular/pom.xml | 56 ++-- beam/pom.xml| 64 + bigquery/pom.xml| 85 cassandra/pom.xml | 54 +--- .../development/writing_zeppelin_interpreter.md | 44 +- elasticsearch/pom.xml | 53 +-- file/pom.xml| 59 +--- flink/pom.xml | 75 ++ geode/pom.xml | 52 +-- groovy/pom.xml | 81 ++- .../org/apache/zeppelin/groovy/GObject.java | 2 +- .../zeppelin/groovy/GroovyInterpreter.java | 2 +- hbase/pom.xml | 53 +-- helium-dev/pom.xml | 51 ++- ignite/pom.xml | 54 +--- interpreter-parent/pom.xml | 138 +++ jdbc/pom.xml| 52 +-- kylin/pom.xml | 56 ++-- lens/pom.xml| 67 + livy/pom.xml| 55 +--- markdown/pom.xml| 53 +-- pig/pom.xml | 55 +--- python/pom.xml | 61 ++-- scalding/pom.xml| 54 +--- scio/pom.xml| 45 ++ shell/pom.xml | 57 ++-- zeppelin-interpreter/pom.xml| 75 ++ .../interpreter/InterpreterSettingManager.java | 11 +- 29 files changed, 456 insertions(+), 1161 deletions(-) -- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/4a3057fa/alluxio/pom.xml -- diff --git a/alluxio/pom.xml b/alluxio/pom.xml index ed6b981..f36494e 100644 --- a/alluxio/pom.xml +++ b/alluxio/pom.xml @@ -20,10 +20,10 @@ 4.0.0 -zeppelin +interpreter-parent org.apache.zeppelin 0.8.0-SNAPSHOT -.. +../interpreter-parent org.apache.zeppelin @@ -34,6 +34,7 @@ 1.0.0 +alluxio @@ -129,54 +130,12 @@ maven-enforcer-plugin - - -enforce -none - - - maven-dependency-plugin - - -copy-dependencies -package - -copy-dependencies - - - ${project.build.directory}/../../interpreter/alluxio -false -false -true
[1/4] zeppelin git commit: ZEPPELIN-2685. Improvement on Interpreter class
Repository: zeppelin Updated Branches: refs/heads/master ed8755d3f -> 9812e26bc http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java index e7a04f3..cf1713f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/EchoInterpreter.java @@ -41,7 +41,7 @@ public class EchoInterpreter extends Interpreter { @Override public InterpreterResult interpret(String st, InterpreterContext context) { -if (Boolean.parseBoolean(property.getProperty("zeppelin.interpreter.echo.fail", "false"))) { +if (Boolean.parseBoolean(getProperty("zeppelin.interpreter.echo.fail", "false"))) { return new InterpreterResult(InterpreterResult.Code.ERROR); } else { return new InterpreterResult(InterpreterResult.Code.SUCCESS, st); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index f3137d9..abbaa35 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -52,10 +52,10 @@ public class InterpreterFactoryTest extends AbstractInterpreterTest { assertEquals(DoubleEchoInterpreter.class.getName(), remoteInterpreter.getClassName()); } - @Test(expected = InterpreterException.class) + @Test public void testUnknownRepl1() throws IOException { interpreterSettingManager.setInterpreterBinding("user1", "note1", interpreterSettingManager.getSettingIds()); -interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl"); +assertNull(interpreterFactory.getInterpreter("user1", "note1", "test.unknown_repl")); } @Test http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java index 0bcdb6f..605476f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java @@ -96,7 +96,7 @@ public class InterpreterSettingManagerTest extends AbstractInterpreterTest { } @Test - public void testCreateUpdateRemoveSetting() throws IOException { + public void testCreateUpdateRemoveSetting() throws IOException, InterpreterException { // create new interpreter setting InterpreterOption option = new InterpreterOption(); option.setPerNote("scoped"); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java index 9deafcf..7a904c6 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SleepInterpreter.java @@ -46,7 +46,7 @@ public class SleepInterpreter extends Interpreter { @Override public Scheduler getScheduler() { -if (Boolean.parseBoolean(property.getProperty("zeppelin.SleepInterpreter.parallel", "false"))) { +if (Boolean.parseBoolean(getProperty("zeppelin.SleepInterpreter.parallel", "false"))) { return SchedulerFactory.singleton().createOrGetParallelScheduler( "Parallel-" + SleepInterpreter.class.getName(), 10); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java -- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/SparkInterpreterModeTest.java b/zeppelin-zengine/src/test/
[4/4] zeppelin git commit: ZEPPELIN-2685. Improvement on Interpreter class
ZEPPELIN-2685. Improvement on Interpreter class ### What is this PR for? Follow up of #2577. Main changes on Interpreter * Add throw `InterpreterException` which is checked exception for the abstract methods of `Interpreter`, this would enforce the interpreter implementation to throw `InterpreterException`. * field name refactoring. * `property` -> `properties` * `getProperty()` --> `getProperties()` * Introduce launcher layer for interpreter launching. Currently we only use shell script to launch interpreter, but it could be any other service or component to launch interpreter, such as livy server , other 3rd party tools or even we may create a separate module for interpreter launcher * abstract cass `InterpreterLauncher` * For now, only 2 implementation: `ShellScriptLauncher` & `SparkInterpreterLauncher`. We could add method in class `Interpreter` to allow interpreter to specify its own launcher class, but it could be future work. ### What type of PR is it? [Improvement | Refactoring] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-2685 ### How should this be tested? Unit test is covered. `ShellScriptLauncherTest` & `SparkInterpreterLauncherTest` ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #2592 from zjffdu/ZEPPELIN-2685 and squashes the following commits: 17dc2f1 [Jeff Zhang] address comments e545cc3 [Jeff Zhang] ZEPPELIN-2685. Improvement on Interpreter class Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/9812e26b Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/9812e26b Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/9812e26b Branch: refs/heads/master Commit: 9812e26bced75033b5795ee0fb8d8c08df9c83b5 Parents: ed8755d Author: Jeff Zhang Authored: Fri Sep 22 15:00:00 2017 +0800 Committer: Jeff Zhang Committed: Sat Oct 14 13:45:51 2017 +0800 -- bin/common.sh | 2 +- bin/interpreter.sh | 1 + docs/interpreter/spark.md | 6 + .../elasticsearch/ElasticsearchInterpreter.java | 8 +- .../apache/zeppelin/file/FileInterpreter.java | 3 +- .../zeppelin/file/HDFSFileInterpreter.java | 2 +- .../apache/zeppelin/flink/FlinkInterpreter.java | 8 +- .../org/apache/zeppelin/groovy/GObject.java | 6 +- .../zeppelin/groovy/GroovyInterpreter.java | 7 +- .../apache/zeppelin/hbase/HbaseInterpreter.java | 4 +- .../zeppelin/hbase/HbaseInterpreterTest.java| 3 +- .../apache/zeppelin/helium/DevInterpreter.java | 3 +- .../zeppelin/ignite/IgniteSqlInterpreter.java | 2 +- .../ignite/IgniteSqlInterpreterTest.java| 3 +- .../apache/zeppelin/jdbc/JDBCInterpreter.java | 25 +- .../zeppelin/jdbc/JDBCInterpreterTest.java | 3 +- .../zeppelin/livy/BaseLivyInterpreter.java | 12 +- .../zeppelin/livy/LivySparkSQLInterpreter.java | 4 +- .../apache/zeppelin/livy/LivyInterpreterIT.java | 23 +- .../org/apache/zeppelin/pig/PigInterpreter.java | 3 +- .../zeppelin/pig/PigQueryInterpreter.java | 4 +- .../zeppelin/pig/PigQueryInterpreterTest.java | 3 +- .../zeppelin/python/IPythonInterpreter.java | 33 +- .../zeppelin/python/PythonCondaInterpreter.java | 27 +- .../python/PythonDockerInterpreter.java | 25 +- .../zeppelin/python/PythonInterpreter.java | 23 +- .../python/PythonInterpreterPandasSql.java | 10 +- .../zeppelin/python/IPythonInterpreterTest.java | 13 +- .../python/PythonCondaInterpreterTest.java | 8 +- .../python/PythonDockerInterpreterTest.java | 6 +- .../python/PythonInterpreterMatplotlibTest.java | 9 +- .../python/PythonInterpreterPandasSqlTest.java | 13 +- .../zeppelin/python/PythonInterpreterTest.java | 9 +- .../zeppelin/scalding/ScaldingInterpreter.java | 8 +- .../apache/zeppelin/shell/ShellInterpreter.java | 4 +- .../zeppelin/spark/IPySparkInterpreter.java | 21 +- .../zeppelin/spark/PySparkInterpreter.java | 45 +- .../apache/zeppelin/spark/SparkInterpreter.java | 20 +- .../zeppelin/spark/SparkRInterpreter.java | 7 +- .../zeppelin/spark/SparkSqlInterpreter.java | 9 +- .../zeppelin/spark/SparkZeppelinContext.java| 8 +- .../org/apache/zeppelin/spark/ZeppelinR.java| 16 +- .../zeppelin/spark/IPySparkInterpreterTest.java | 5 +- .../spark/PySparkInterpreterMatplotlibTest.java | 12 +- .../zeppelin/spark/PySparkInterpreterTest.java | 15 +- .../zeppelin/spark/SparkInterpreterTest.java| 10 +- .../zeppelin/spark/SparkSqlInterpreterTest.java | 8 +- zeppelin-interpreter/pom.xml| 5 +
[2/4] zeppelin git commit: ZEPPELIN-2685. Improvement on Interpreter class
@Override -void checkType(String value) throws Exception { - Integer.valueOf(value); -} - }, - LONG { -@Override -void checkType(String value) throws Exception { - Long.valueOf(value); -} - }, - FLOAT { -@Override -void checkType(String value) throws Exception { - Float.valueOf(value); -} - }, - BOOLEAN { -@Override -void checkType(String value) throws Exception { - Boolean.valueOf(value); -} - }; - - boolean isType(String value) { -try { - checkType(value); -} catch (Exception e) { - LOG.error("Exception in ZeppelinConfiguration while isType", e); - return false; -} -return true; - } - - String typeString() { -return name().toUpperCase(); - } - - abstract void checkType(String value) throws Exception; -} - } -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index f020919..7233239 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -77,7 +77,7 @@ public class InterpreterFactory { return interpreter; } } - throw new InterpreterException(replName + " interpreter not found"); + return null; } else { // first assume replName is 'name' of interpreter. ('groupName' is ommitted) http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java -- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 5af01dc..a82d5bf 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -18,7 +18,6 @@ package org.apache.zeppelin.interpreter; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; @@ -34,19 +33,22 @@ import org.apache.zeppelin.dep.DependencyResolver; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.AngularObjectRegistryListener; import org.apache.zeppelin.helium.ApplicationEventListener; +import org.apache.zeppelin.interpreter.launcher.InterpreterLaunchContext; +import org.apache.zeppelin.interpreter.launcher.InterpreterLauncher; +import org.apache.zeppelin.interpreter.launcher.ShellScriptLauncher; +import org.apache.zeppelin.interpreter.launcher.SparkInterpreterLauncher; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess; +import org.apache.zeppelin.interpreter.remote.RemoteInterpreterEventPoller; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.interpreter.remote.RemoteInterpreterRunningProcess; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileNotFoundException; -import java.io.FilenameFilter; +import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URL; @@ -58,7 +60,6 @@ import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -132,6 +133,10 @@ public class InterpreterSetting { private transient ZeppelinConfiguration conf = new ZeppelinConfiguration(); + // TODO(zjffdu) ShellScriptLauncher is the only launcher implemention for now. It could be other + // launcher in future when we have other launcher implementation. e.g. third party launcher + // service like livy + private transient InterpreterLauncher launcher;
[3/4] zeppelin git commit: ZEPPELIN-2685. Improvement on Interpreter class
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 18da034..5a5110f 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -353,7 +353,7 @@ public class SparkInterpreter extends Interpreter { public boolean isYarnMode() { String master = getProperty("master"); if (master == null) { - master = getProperty().getProperty("spark.master", "local[*]"); + master = getProperty("spark.master", "local[*]"); } return master.startsWith("yarn"); } @@ -376,7 +376,7 @@ public class SparkInterpreter extends Interpreter { } conf.set("spark.scheduler.mode", "FAIR"); -Properties intpProperty = getProperty(); +Properties intpProperty = getProperties(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); @@ -509,7 +509,7 @@ public class SparkInterpreter extends Interpreter { } conf.set("spark.scheduler.mode", "FAIR"); -Properties intpProperty = getProperty(); +Properties intpProperty = getProperties(); for (Object k : intpProperty.keySet()) { String key = (String) k; String val = toString(intpProperty.get(key)); @@ -543,19 +543,19 @@ public class SparkInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { this.enableSupportedVersionCheck = java.lang.Boolean.parseBoolean( -property.getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); +getProperty("zeppelin.spark.enableSupportedVersionCheck", "true")); // set properties and do login before creating any spark stuff for secured cluster if (isYarnMode()) { System.setProperty("SPARK_YARN_MODE", "true"); } -if (getProperty().containsKey("spark.yarn.keytab") && -getProperty().containsKey("spark.yarn.principal")) { +if (getProperties().containsKey("spark.yarn.keytab") && +getProperties().containsKey("spark.yarn.principal")) { try { -String keytab = getProperty().getProperty("spark.yarn.keytab"); -String principal = getProperty().getProperty("spark.yarn.principal"); +String keytab = getProperties().getProperty("spark.yarn.keytab"); +String principal = getProperties().getProperty("spark.yarn.principal"); UserGroupInformation.loginUserFromKeytab(principal, keytab); } catch (IOException e) { throw new RuntimeException("Can not pass kerberos authentication", e); @@ -963,7 +963,7 @@ public class SparkInterpreter extends Interpreter { sparkUrl = getSparkUIUrl(); Map infos = new java.util.HashMap<>(); infos.put("url", sparkUrl); -String uiEnabledProp = property.getProperty("spark.ui.enabled", "true"); +String uiEnabledProp = getProperty("spark.ui.enabled", "true"); java.lang.Boolean uiEnabled = java.lang.Boolean.parseBoolean( uiEnabledProp.trim()); if (!uiEnabled) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/9812e26b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java -- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java index ca52f79..1bdd4dc 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkRInterpreter.java @@ -54,7 +54,7 @@ public class SparkRInterpreter extends Interpreter { } @Override - public void open() { + public void open() throws InterpreterException { String rCmdPath = getProperty("zeppelin.R.cmd"); String sparkRLibPath; @@ -105,7 +105,8 @@ public class SparkRInterpreter extends Interpreter { } @Override - public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) { + public InterpreterResult interpret(String lines, InterpreterContext interpreterContext) + throws InterpreterException { SparkInterpreter sparkInterpreter = getSparkInterpreter(); sparkInterpreter.populateSparkWebUrl(interpreterContext); @@ -220,7 +221,7 @@ public class SparkRInterpreter extends Interpreter { return new ArrayList<>(); } - private SparkInterpreter getSparkInterpreter() { + private SparkInterpreter getSparkInterpreter() throws InterpreterException { LazyOpenInterpreter lazy = null; SparkInterpreter spark = null; Interpreter p = getInterpreterInTheSameSessionByCl