Repository: zeppelin Updated Branches: refs/heads/branch-0.6 7db07f227 -> d962297c2
Revert "merge master into branch-0.6" This reverts commit 7db07f22742896c6de0691cacbd37dbe5b0b493e. Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/bc75d6ba Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/bc75d6ba Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/bc75d6ba Branch: refs/heads/branch-0.6 Commit: bc75d6bad9779ede8f03ab59efcbeaca614de07b Parents: 7db07f2 Author: Prabhjyot Singh <[email protected]> Authored: Sat Jul 2 19:43:35 2016 +0530 Committer: Prabhjyot Singh <[email protected]> Committed: Sat Jul 2 19:43:35 2016 +0530 ---------------------------------------------------------------------- .../org/apache/zeppelin/livy/LivyHelper.java | 4 +-- .../zeppelin/livy/LivySparkSQLInterpreter.java | 35 ++++++++++---------- .../src/main/resources/interpreter-setting.json | 5 --- 3 files changed, 19 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java index ec77f1a..2c66fa9 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java @@ -96,7 +96,7 @@ public class LivyHelper { }.getType()); if (jsonMap.get("state").equals("idle")) { break; - } else if (jsonMap.get("state").equals("error") || jsonMap.get("state").equals("dead")) { + } else if (jsonMap.get("state").equals("error")) { json = executeHTTP(property.getProperty("zeppelin.livy.url") + "/sessions/" + sessionId + "/log", "GET", null, @@ -124,7 +124,7 @@ public class LivyHelper { protected void initializeSpark(final InterpreterContext context, final Map<String, Integer> userSessionMap) throws Exception { - interpret("val sqlContext = new org.apache.spark.sql.SQLContext(sc)\n" + + interpret("val sqlContext= new org.apache.spark.sql.SQLContext(sc)\n" + "import sqlContext.implicits._", context, userSessionMap); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java ---------------------------------------------------------------------- diff --git a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java index 22773df..3c60204 100644 --- a/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java +++ b/livy/src/main/java/org/apache/zeppelin/livy/LivySparkSQLInterpreter.java @@ -35,6 +35,20 @@ import java.util.Properties; public class LivySparkSQLInterpreter extends Interpreter { Logger LOGGER = LoggerFactory.getLogger(LivySparkSQLInterpreter.class); + static String DEFAULT_MAX_RESULT = "1000"; + + static { + Interpreter.register( + "sql", + "livy", + LivySparkSQLInterpreter.class.getName(), + new InterpreterPropertyBuilder() + .add("zeppelin.livy.spark.maxResult", + DEFAULT_MAX_RESULT, + "Max number of SparkSQL result to display.") + .build() + ); + } protected Map<String, Integer> userSessionMap; private LivyHelper livyHelper; @@ -80,7 +94,7 @@ public class LivySparkSQLInterpreter extends Interpreter { line.replaceAll("\"", "\\\\\"") .replaceAll("\\n", " ") + "\").show(" + - property.get("zeppelin.livy.spark.sql.maxResult") + ")", + property.get("zeppelin.livy.spark.maxResult") + ")", interpreterContext, userSessionMap); if (res.code() == InterpreterResult.Code.SUCCESS) { @@ -123,10 +137,6 @@ public class LivySparkSQLInterpreter extends Interpreter { } } - public boolean concurrentSQL() { - return Boolean.parseBoolean(getProperty("zeppelin.livy.concurrentSQL")); - } - @Override public void cancel(InterpreterContext context) { livyHelper.cancelHTTP(context.getParagraphId()); @@ -144,19 +154,8 @@ public class LivySparkSQLInterpreter extends Interpreter { @Override public Scheduler getScheduler() { - if (concurrentSQL()) { - int maxConcurrency = 10; - return SchedulerFactory.singleton().createOrGetParallelScheduler( - LivySparkInterpreter.class.getName() + this.hashCode(), maxConcurrency); - } else { - Interpreter intp = - getInterpreterInTheSameSessionByClassName(LivySparkInterpreter.class.getName()); - if (intp != null) { - return intp.getScheduler(); - } else { - return null; - } - } + return SchedulerFactory.singleton().createOrGetFIFOScheduler( + LivySparkInterpreter.class.getName() + this.hashCode()); } @Override http://git-wip-us.apache.org/repos/asf/zeppelin/blob/bc75d6ba/livy/src/main/resources/interpreter-setting.json ---------------------------------------------------------------------- diff --git a/livy/src/main/resources/interpreter-setting.json b/livy/src/main/resources/interpreter-setting.json index e90216c..7ae435f 100644 --- a/livy/src/main/resources/interpreter-setting.json +++ b/livy/src/main/resources/interpreter-setting.json @@ -87,11 +87,6 @@ "propertyName": "zeppelin.livy.spark.sql.maxResult", "defaultValue": "1000", "description": "Max number of SparkSQL result to display." - }, - "zeppelin.livy.concurrentSQL": { - "propertyName": "zeppelin.livy.concurrentSQL", - "defaultValue": "false", - "description": "Execute multiple SQL concurrently if set true." } } },
