This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new b9006a2 ZEPPELIN-4227. Move StandardInterpreterLauncher and
SparkInterpreterLauncher into zeppelin-zengine
b9006a2 is described below
commit b9006a247043aa3151a798d10c85b7edf6ea9fe3
Author: Jeff Zhang <[email protected]>
AuthorDate: Sun Jul 7 22:25:17 2019 +0800
ZEPPELIN-4227. Move StandardInterpreterLauncher and
SparkInterpreterLauncher into zeppelin-zengine
### What is this PR for?
We should include these default interpreter launcher into zeppelin-zengine,
so that user can use zeppelin even without building zeppelin-plugins module.
### What type of PR is it?
[Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4227
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <[email protected]>
Closes #3421 from zjffdu/ZEPPELIN-4227 and squashes the following commits:
ee1c7bcc1 [Jeff Zhang] ZEPPELIN-4227. Move StandardInterpreterLauncher and
SparkInterpreterLauncher into zeppelin-zengine
---
.travis.yml | 4 +-
zeppelin-plugins/launcher/cluster/pom.xml | 7 +--
zeppelin-plugins/launcher/docker/pom.xml | 5 --
zeppelin-plugins/launcher/spark/pom.xml | 67 ----------------------
zeppelin-plugins/launcher/standard/pom.xml | 49 ----------------
zeppelin-plugins/pom.xml | 2 -
.../apache/zeppelin/rest/NotebookRestApiTest.java | 4 +-
.../launcher/SparkInterpreterLauncher.java | 0
.../launcher/StandardInterpreterLauncher.java | 0
.../org/apache/zeppelin/plugin/PluginManager.java | 13 +++++
.../launcher/SparkInterpreterLauncherTest.java | 9 +++
.../launcher/StandardInterpreterLauncherTest.java | 0
12 files changed, 27 insertions(+), 133 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index ef2193a..2838e1b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,7 +86,7 @@ matrix:
- sudo: required
jdk: "openjdk8"
dist: xenial
- env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev
-Pexamples -Pspark-scala-2.11" BUILD_FLAG="clean install -Pbuild-distr
-DskipRat" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl
${INTERPRETERS}"
TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
+ env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev
-Pexamples -Pspark-scala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat
-DskipTests" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl
${INTERPRETERS}"
TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
# Test selenium with spark module for spark 2.3
- jdk: "openjdk8"
@@ -98,7 +98,7 @@ matrix:
# Test interpreter modules
- jdk: "openjdk8"
dist: xenial
- env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0"
PROFILE="-Pscala-2.10" BUILD_FLAG="clean install -DskipTests -DskipRat -am"
TEST_FLAG="test -DskipRat" MODULES="-pl $(echo
.,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed
's/!//g')" TEST_PROJECTS=""
+ env: PYTHON="3" SPARKR="true" SCALA_VER="2.10" TENSORFLOW="1.0.0"
PROFILE="-Pscala-2.10" BUILD_FLAG="install -DskipTests -DskipRat -am"
TEST_FLAG="test -DskipRat" MODULES="-pl $(echo
.,zeppelin-interpreter,zeppelin-interpreter-api,${INTERPRETERS} | sed
's/!//g')" TEST_PROJECTS=""
# Run Spark integration test and unit test separately for each spark
version
diff --git a/zeppelin-plugins/launcher/cluster/pom.xml
b/zeppelin-plugins/launcher/cluster/pom.xml
index bf24546..08b03be 100644
--- a/zeppelin-plugins/launcher/cluster/pom.xml
+++ b/zeppelin-plugins/launcher/cluster/pom.xml
@@ -39,15 +39,10 @@
<properties>
<plugin.name>Launcher/ClusterInterpreterLauncher</plugin.name>
</properties>
-
+
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
- <artifactId>launcher-standard</artifactId>
- <version>0.9.0-SNAPSHOT</version>
- </dependency>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
<artifactId>launcher-docker</artifactId>
<version>0.9.0-SNAPSHOT</version>
</dependency>
diff --git a/zeppelin-plugins/launcher/docker/pom.xml
b/zeppelin-plugins/launcher/docker/pom.xml
index 6d029fe..b9a47cb 100644
--- a/zeppelin-plugins/launcher/docker/pom.xml
+++ b/zeppelin-plugins/launcher/docker/pom.xml
@@ -42,11 +42,6 @@
<dependencies>
<dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>launcher-spark</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
<groupId>com.hubspot.jinjava</groupId>
<artifactId>jinjava</artifactId>
<version>2.4.12</version>
diff --git a/zeppelin-plugins/launcher/spark/pom.xml
b/zeppelin-plugins/launcher/spark/pom.xml
deleted file mode 100644
index 189c27f..0000000
--- a/zeppelin-plugins/launcher/spark/pom.xml
+++ /dev/null
@@ -1,67 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd";>
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>zengine-plugins-parent</artifactId>
- <groupId>org.apache.zeppelin</groupId>
- <version>0.9.0-SNAPSHOT</version>
- <relativePath>../../../zeppelin-plugins</relativePath>
- </parent>
-
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>launcher-spark</artifactId>
- <packaging>jar</packaging>
- <version>0.9.0-SNAPSHOT</version>
- <name>Zeppelin: Plugin SparkInterpreterLauncher</name>
- <description>Spark Launcher implementation based on shell script
interpreter.sh</description>
-
- <properties>
- <plugin.name>Launcher/SparkInterpreterLauncher</plugin.name>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>launcher-standard</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- </plugin>
-
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>${plugin.surefire.version}</version>
- <configuration>
- <environmentVariables>
-
<ZEPPELIN_HOME>${project.basedir}/../../..</ZEPPELIN_HOME>
- </environmentVariables>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/zeppelin-plugins/launcher/standard/pom.xml
b/zeppelin-plugins/launcher/standard/pom.xml
deleted file mode 100644
index c8c68ce..0000000
--- a/zeppelin-plugins/launcher/standard/pom.xml
+++ /dev/null
@@ -1,49 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd";>
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>zengine-plugins-parent</artifactId>
- <groupId>org.apache.zeppelin</groupId>
- <version>0.9.0-SNAPSHOT</version>
- <relativePath>../../../zeppelin-plugins</relativePath>
- </parent>
-
- <groupId>org.apache.zeppelin</groupId>
- <artifactId>launcher-standard</artifactId>
- <packaging>jar</packaging>
- <version>0.9.0-SNAPSHOT</version>
- <name>Zeppelin: Plugin StandardLauncher</name>
- <description>Launcher implementation based on shell script
interpreter.sh</description>
-
- <properties>
- <plugin.name>Launcher/StandardInterpreterLauncher</plugin.name>
- </properties>
-
- <build>
- <plugins>
- <plugin>
- <artifactId>maven-dependency-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/zeppelin-plugins/pom.xml b/zeppelin-plugins/pom.xml
index dea3a47..bca6421 100644
--- a/zeppelin-plugins/pom.xml
+++ b/zeppelin-plugins/pom.xml
@@ -43,9 +43,7 @@
<module>notebookrepo/zeppelin-hub</module>
<module>notebookrepo/filesystem</module>
- <module>launcher/standard</module>
<module>launcher/k8s-standard</module>
- <module>launcher/spark</module>
<module>launcher/cluster</module>
<module>launcher/docker</module>
</modules>
diff --git
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
index 1835b51..079fe5f 100644
---
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
+++
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java
@@ -422,8 +422,8 @@ public class NotebookRestApiTest extends
AbstractTestRestApi {
assertEquals(resp.get("status"), "OK");
post2.releaseConnection();
- assertEquals(Job.Status.FINISHED, p1.getStatus());
- assertEquals(Job.Status.FINISHED, p2.getStatus());
+ assertEquals(p1.getReturn().toString(), Job.Status.FINISHED,
p1.getStatus());
+ assertEquals(p2.getReturn().toString(), Job.Status.FINISHED,
p2.getStatus());
assertNotNull(p2.getReturn());
assertEquals("abc\n", p2.getReturn().message().get(0).getData());
} finally {
diff --git
a/zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
similarity index 100%
rename from
zeppelin-plugins/launcher/spark/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
rename to
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
diff --git
a/zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
similarity index 100%
rename from
zeppelin-plugins/launcher/standard/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
rename to
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncher.java
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
index c3a2b9b..cdcfed0 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/plugin/PluginManager.java
@@ -140,6 +140,19 @@ public class PluginManager {
return cachedLaunchers.get(launcherPlugin);
}
LOGGER.info("Loading Interpreter Launcher Plugin: " + launcherPlugin);
+ // load plugin from classpath directly first for these builtin
InterpreterLauncher.
+ // If fails, then try to load it from plugin folder.
+ try {
+ InterpreterLauncher launcher = (InterpreterLauncher)
+ (Class.forName("org.apache.zeppelin.interpreter.launcher." +
launcherPlugin))
+ .getConstructor(ZeppelinConfiguration.class,
RecoveryStorage.class)
+ .newInstance(zConf, recoveryStorage);
+ return launcher;
+ } catch (InstantiationException | IllegalAccessException |
ClassNotFoundException
+ | NoSuchMethodException | InvocationTargetException e) {
+ LOGGER.warn("Fail to instantiate InterpreterLauncher from classpath
directly:" + launcherPlugin, e);
+ }
+
URLClassLoader pluginClassLoader = getPluginClassLoader(pluginsDir,
"Launcher", launcherPlugin);
String pluginClass = "org.apache.zeppelin.interpreter.launcher." +
launcherPlugin;
InterpreterLauncher launcher = null;
diff --git
a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
similarity index 97%
rename from
zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
rename to
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index 118e7d4..8500de6 100644
---
a/zeppelin-plugins/launcher/spark/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -25,7 +25,10 @@ import
org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess;
import org.apache.zeppelin.util.Util;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -37,6 +40,8 @@ import static org.junit.Assert.assertTrue;
public class SparkInterpreterLauncherTest {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(SparkInterpreterLauncher.class);
+
private String sparkHome;
private String zeppelinHome;
@@ -47,7 +52,11 @@ public class SparkInterpreterLauncherTest {
}
sparkHome = DownloadUtils.downloadSpark("2.3.2");
+
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(),
+ new File("..").getAbsolutePath());
+
zeppelinHome = ZeppelinConfiguration.create().getZeppelinHome();
+ LOGGER.info("ZEPPELIN_HOME: " + zeppelinHome);
}
@Test
diff --git
a/zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
similarity index 100%
rename from
zeppelin-plugins/launcher/standard/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
rename to
zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/StandardInterpreterLauncherTest.java
