This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new e909600 ZEPPELIN-3935. Enable color output of spark scala interpreter
e909600 is described below
commit e909600b437c80a1b4eaaea63e8040b283ed4cf2
Author: jeffzhang.zjf <[email protected]>
AuthorDate: Thu Jan 24 17:07:49 2019 +0800
ZEPPELIN-3935. Enable color output of spark scala interpreter
### What is this PR for?
This is a simple PR to enable the color output of spark scala interpreter.
See the screenshot for more details
### What type of PR is it?
[Bug Fix | Improvement | Feature | Documentation | Hot Fix | Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-3935
### How should this be tested?
* Tested manually
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: jeffzhang.zjf <[email protected]>
Closes #3289 from zjffdu/ZEPPELIN-3935 and squashes the following commits:
d3c6f10b5 [jeffzhang.zjf] ZEPPELIN-3935. Enable color output of spark scala
interpreter
---
docs/interpreter/spark.md | 4 ++++
.../java/org/apache/zeppelin/spark/SparkInterpreter.java | 4 ++++
.../src/main/resources/interpreter-setting.json | 7 +++++++
.../apache/zeppelin/spark/NewSparkInterpreterTest.java | 16 ++++++++++++++++
.../apache/zeppelin/spark/OldSparkInterpreterTest.java | 2 ++
.../apache/zeppelin/rest/ZeppelinSparkClusterTest.java | 2 ++
6 files changed, 35 insertions(+)
diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md
index 784d749..9140825 100644
--- a/docs/interpreter/spark.md
+++ b/docs/interpreter/spark.md
@@ -160,6 +160,10 @@ You can also set other Spark properties which are not
listed in the table. For a
<td></td>
<td>Overrides Spark UI default URL. Value should be a full URL (ex:
http://{hostName}/{uniquePath}</td>
</tr>
+ <td>zeppelin.spark.scala.color</td>
+ <td>true</td>
+ <td>Whether to enable color output of spark scala interpreter</td>
+ </tr>
</table>
Without any configuration, Spark interpreter works out of box in local mode.
But if you want to connect to your Spark cluster, you'll need to follow below
two simple steps.
diff --git
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
index dabe9ed..4a9a9de 100644
---
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
+++
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java
@@ -45,6 +45,10 @@ public class SparkInterpreter extends
AbstractSparkInterpreter {
public SparkInterpreter(Properties properties) {
super(properties);
+ // set scala.color
+ if
(Boolean.parseBoolean(properties.getProperty("zeppelin.spark.scala.color",
"true"))) {
+ System.setProperty("scala.color", "true");
+ }
if (Boolean.parseBoolean(properties.getProperty("zeppelin.spark.useNew",
"false"))) {
delegation = new NewSparkInterpreter(properties);
} else {
diff --git a/spark/interpreter/src/main/resources/interpreter-setting.json
b/spark/interpreter/src/main/resources/interpreter-setting.json
index 60224ce..5ff5cfe 100644
--- a/spark/interpreter/src/main/resources/interpreter-setting.json
+++ b/spark/interpreter/src/main/resources/interpreter-setting.json
@@ -88,6 +88,13 @@
"defaultValue": false,
"description": "Whether to hide spark ui in zeppelin ui",
"type": "checkbox"
+ },
+ "zeppelin.spark.scala.color": {
+ "envName": null,
+ "propertyName": "zeppelin.spark.scala.color",
+ "defaultValue": true,
+ "description": "Whether to enable color output of spark scala
interpreter",
+ "type": "checkbox"
}
},
"editor": {
diff --git
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
index d69b2f9..54835ae 100644
---
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
+++
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/NewSparkInterpreterTest.java
@@ -84,6 +84,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.uiWebUrl", "fake_spark_weburl");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterContext context = InterpreterContext.builder()
.setInterpreterOut(new InterpreterOutput(null))
@@ -375,6 +377,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("spark.app.name", "test");
properties.setProperty("zeppelin.spark.maxResult", "100");
properties.setProperty("zeppelin.spark.useNew", "true");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
// download spark-avro jar
URL website = new
URL("http://repo1.maven.org/maven2/com/databricks/spark-avro_2.11/3.2.0/spark-avro_2.11-3.2.0.jar";);
@@ -404,6 +408,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.dep.localrepo",
Files.createTempDir().getAbsolutePath());
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterGroup intpGroup = new InterpreterGroup();
interpreter = new SparkInterpreter(properties);
@@ -433,6 +439,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.printREPLOutput", "false");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
InterpreterContext.set(getInterpreterContext());
interpreter = new SparkInterpreter(properties);
@@ -460,6 +468,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("spark.scheduler.mode", "FAIR");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@@ -489,6 +499,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("spark.ui.enabled", "false");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@@ -514,6 +526,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
properties.setProperty("zeppelin.spark.ui.hidden", "true");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
interpreter = new SparkInterpreter(properties);
assertTrue(interpreter.getDelegation() instanceof NewSparkInterpreter);
@@ -537,6 +551,8 @@ public class NewSparkInterpreterTest {
properties.setProperty("zeppelin.spark.maxResult", "100");
properties.setProperty("zeppelin.spark.test", "true");
properties.setProperty("zeppelin.spark.useNew", "true");
+ // disable color output for easy testing
+ properties.setProperty("zeppelin.spark.scala.color", "false");
SparkInterpreter interpreter1 = new SparkInterpreter(properties);
SparkInterpreter interpreter2 = new SparkInterpreter(properties);
diff --git
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java
index 8ae66b2..a435272 100644
---
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java
+++
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/OldSparkInterpreterTest.java
@@ -85,6 +85,8 @@ public class OldSparkInterpreterTest {
p.setProperty("zeppelin.spark.importImplicit", "true");
p.setProperty("zeppelin.dep.localrepo",
tmpDir.newFolder().getAbsolutePath());
p.setProperty("zeppelin.spark.property_1", "value_1");
+ // disable color output for easy testing
+ p.setProperty("zeppelin.spark.scala.color", "false");
return p;
}
diff --git
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
index de2c061..d3516bd 100644
---
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
+++
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
@@ -122,6 +122,8 @@ public class ZeppelinSparkClusterTest extends
AbstractTestRestApi {
new InterpreterProperty("zeppelin.spark.test", "true"));
sparkProperties.put("spark.serializer",
new InterpreterProperty("spark.serializer",
"org.apache.spark.serializer.KryoSerializer"));
+ sparkProperties.put("zeppelin.spark.scala.color",
+ new InterpreterProperty("zeppelin.spark.scala.color", "false"));
TestUtils.getInstance(Notebook.class).getInterpreterSettingManager().restart(sparkIntpSetting.getId());
}
