Repository: zeppelin
Updated Branches:
refs/heads/master 89cf8262e -> c484619d1
[ZEPPELIN-1482] Load updated dependency library on interpreter restart
### What is this PR for?
If user specifies library path in interpreter dependency setting, even when the
file on this path is updated, new file doesn't take effect on interpreter
_restart_ but does only when user _clicks Edit -> Save._
The mechanism of dependency loading is copying all dependency libraries under
`local-repo/
{interpreterId}` and add these directory to classpath of interpreter process.
Zeppelin copies these dependencies either on Zeppelin startup or dependency
saving/editing.
This PR checks if the library on specified local path is updated, and copy them
to `local-repo/
{interpreterId}` on restart if there is change.
### What type of PR is it?
Bug Fix & Improvement
### What is the Jira issue?
[ZEPPELIN-1482](https://issues.apache.org/jira/browse/ZEPPELIN-1482)
### How should this be tested?
1. Download commons-csv-1.1.jar and commons-csv-1.2.jar to /my/path
2. cp commons-csv-1.2.jar /my/path/commons-csv.jar
3. Set dependency artifact of spark interpreter to /my/path/commons-csv.jar
4. Run `%spark import org.apache.commons.csv.CSVFormat.Predefined` in paragraph
and see if it runs without error
5. cp commons-csv-1.1.jar /my/path/commons-csv.jar
6. Restart spark interpreter
7. Run `%spark import org.apache.commons.csv.CSVFormat.Predefined` in paragraph
and see if error occurs.
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Mina Lee <[email protected]>
Closes #1453 from minahlee/ZEPPELIN-1482 and squashes the following commits:
ea11664 [Mina Lee] Check if dependency library on specified path has changed
and copy them under local-repo/{interpreterId} on interpreter restart
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/c484619d
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/c484619d
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/c484619d
Branch: refs/heads/master
Commit: c484619d19f53fb522a9cb5a05c83192f49961e4
Parents: 89cf826
Author: Mina Lee <[email protected]>
Authored: Fri Sep 23 19:29:21 2016 +0900
Committer: Mina Lee <[email protected]>
Committed: Tue Sep 27 10:39:54 2016 +0900
----------------------------------------------------------------------
.../apache/zeppelin/dep/DependencyResolver.java | 15 ++++++
.../interpreter/InterpreterFactory.java | 50 ++++++++++++++++++--
2 files changed, 61 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c484619d/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
----------------------------------------------------------------------
diff --git
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
index 214175a..87d9178 100644
---
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
+++
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
@@ -104,6 +104,21 @@ public class DependencyResolver extends
AbstractDependencyResolver {
return libs;
}
+ public synchronized void copyLocalDependency(String srcPath, File destPath)
+ throws IOException {
+ if (StringUtils.isBlank(srcPath)) {
+ return;
+ }
+
+ File srcFile = new File(srcPath);
+ File destFile = new File(destPath, srcFile.getName());
+
+ if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
+ FileUtils.copyFile(srcFile, destFile);
+ logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
+ }
+ }
+
private List<File> loadFromMvn(String artifact, Collection<String> excludes)
throws RepositoryException {
Collection<String> allExclusions = new LinkedList<String>();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/c484619d/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index 40dcc30..58adb48 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -197,10 +197,10 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
Set<String> interpreterKeys =
Interpreter.registeredInterpreters.keySet();
for (String interpreterKey : interpreterKeys) {
if (className
-
.equals(Interpreter.registeredInterpreters.get(interpreterKey)
- .getClassName())) {
+
.equals(Interpreter.registeredInterpreters.get(interpreterKey)
+ .getClassName())) {
Interpreter.registeredInterpreters.get(interpreterKey)
- .setPath(interpreterDirString);
+ .setPath(interpreterDirString);
logger.info("Interpreter " + interpreterKey + " found.
class=" + className);
cleanCl.put(interpreterDirString, ccl);
}
@@ -431,7 +431,6 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
}
private void loadInterpreterDependencies(final InterpreterSetting setting) {
-
setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
interpreterSettings.put(setting.getId(), setting);
synchronized (interpreterSettings) {
@@ -477,6 +476,46 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
}
}
+ /**
+ * Overwrite dependency jar under local-repo/{interpreterId}
+ * if jar file in original path is changed
+ */
+ private void copyDependenciesFromLocalPath(final InterpreterSetting setting)
{
+ setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
+ interpreterSettings.put(setting.getId(), setting);
+ synchronized (interpreterSettings) {
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new
File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+ depResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.getId()));
+ }
+ }
+ }
+ setting.setStatus(InterpreterSetting.Status.READY);
+ } catch (Exception e) {
+ logger.error(String.format("Error while copying deps for
interpreter group : %s," +
+ " go to interpreter setting page click on edit and save it
again to make " +
+ "this interpreter work properly.",
+ setting.getGroup()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+ interpreterSettings.put(setting.getId(), setting);
+ }
+ }
+ };
+ t.start();
+ }
+ }
+
private void saveToFile() throws IOException {
String jsonString;
@@ -900,6 +939,9 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
public void restart(String id) {
synchronized (interpreterSettings) {
InterpreterSetting intpsetting = interpreterSettings.get(id);
+ // Check if dependency in specified path is changed
+ // If it did, overwrite old dependency jar with new one
+ copyDependenciesFromLocalPath(intpsetting);
if (intpsetting != null) {
stopJobAllInterpreter(intpsetting);
