Repository: zeppelin
Updated Branches:
refs/heads/branch-0.6 bd750988a -> 65f26bdde
[ZEPPELIN-1482] Load updated dependency library on interpreter restart
If user specifies library path in interpreter dependency setting, even when the
file on this path is updated, new file doesn't take effect on interpreter
_restart_ but does only when user _clicks Edit -> Save._
The mechanism of dependency loading is copying all dependency libraries under
`local-repo/
{interpreterId}` and add these directory to classpath of interpreter process.
Zeppelin copies these dependencies either on Zeppelin startup or dependency
saving/editing.
This PR checks if the library on specified local path is updated, and copy them
to `local-repo/
{interpreterId}` on restart if there is change.
Bug Fix & Improvement
[ZEPPELIN-1482](https://issues.apache.org/jira/browse/ZEPPELIN-1482)
1. Download commons-csv-1.1.jar and commons-csv-1.2.jar to /my/path
2. cp commons-csv-1.2.jar /my/path/commons-csv.jar
3. Set dependency artifact of spark interpreter to /my/path/commons-csv.jar
4. Run `%spark import org.apache.commons.csv.CSVFormat.Predefined` in paragraph
and see if it runs without error
5. cp commons-csv-1.1.jar /my/path/commons-csv.jar
6. Restart spark interpreter
7. Run `%spark import org.apache.commons.csv.CSVFormat.Predefined` in paragraph
and see if error occurs.
* Does the licenses files need update? no
* Is there breaking changes for older versions? no
* Does this needs documentation? no
Author: Mina Lee <[email protected]>
Closes #1453 from minahlee/ZEPPELIN-1482 and squashes the following commits:
ea11664 [Mina Lee] Check if dependency library on specified path has changed
and copy them under local-repo/{interpreterId} on interpreter restart
(cherry picked from commit c484619d19f53fb522a9cb5a05c83192f49961e4)
Signed-off-by: Mina Lee <[email protected]>
Conflicts:
zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/65f26bdd
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/65f26bdd
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/65f26bdd
Branch: refs/heads/branch-0.6
Commit: 65f26bdde0a63c7656bd904cc364454f33a89e64
Parents: bd75098
Author: Mina Lee <[email protected]>
Authored: Fri Sep 23 19:29:21 2016 +0900
Committer: Mina Lee <[email protected]>
Committed: Tue Sep 27 11:43:13 2016 +0900
----------------------------------------------------------------------
.../apache/zeppelin/dep/DependencyResolver.java | 15 +++++++++++
.../interpreter/InterpreterFactory.java | 26 ++++++++++++++++++++
2 files changed, 41 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/65f26bdd/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
----------------------------------------------------------------------
diff --git
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
index 214175a..87d9178 100644
---
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
+++
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java
@@ -104,6 +104,21 @@ public class DependencyResolver extends
AbstractDependencyResolver {
return libs;
}
+ public synchronized void copyLocalDependency(String srcPath, File destPath)
+ throws IOException {
+ if (StringUtils.isBlank(srcPath)) {
+ return;
+ }
+
+ File srcFile = new File(srcPath);
+ File destFile = new File(destPath, srcFile.getName());
+
+ if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) {
+ FileUtils.copyFile(srcFile, destFile);
+ logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath);
+ }
+ }
+
private List<File> loadFromMvn(String artifact, Collection<String> excludes)
throws RepositoryException {
Collection<String> allExclusions = new LinkedList<String>();
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/65f26bdd/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
----------------------------------------------------------------------
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
index 9f30f50..c3b18ba 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java
@@ -366,6 +366,29 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
}
}
+ /**
+ * Overwrite dependency jar under local-repo/{interpreterId}
+ * if jar file in original path is changed
+ */
+ private void copyDependenciesFromLocalPath(final InterpreterSetting setting)
{
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new
File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+ try {
+ depResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.id()));
+ } catch (IOException e) {
+ logger.error("Failed to copy {} to {}",
d.getGroupArtifactVersion(), destDir);
+ }
+ }
+ }
+ }
+ }
+
private void saveToFile() throws IOException {
String jsonString;
@@ -769,6 +792,9 @@ public class InterpreterFactory implements
InterpreterGroupFactory {
public void restart(String id) {
synchronized (interpreterSettings) {
InterpreterSetting intpsetting = interpreterSettings.get(id);
+ // Check if dependency in specified path is changed
+ // If it did, overwrite old dependency jar with new one
+ copyDependenciesFromLocalPath(intpsetting);
if (intpsetting != null) {
stopJobAllInterpreter(intpsetting);
