Repository: incubator-zeppelin Updated Branches: refs/heads/master fff175f92 -> 86a1a1994
[ZEPPELIN-357] Add support for a configurable list of repo for dependencies Make it configurable via `zeppelin.dep.additionalRemoteRepository`, with spark-packages default. This enables notebook to load dependencies with having to add spark-packages each time. There is a fair bit of overlap between `DependencyResolver` and `DependencyContext`, but it appears in order for any custom repo available it needs to be in both components. We should discuss how best to address overlaps between the two components. corneadoug Leemoonsoo Author: Felix Cheung <[email protected]> Closes #368 from felixcheung/sparkpackage and squashes the following commits: 207ffb8 [Felix Cheung] Update test - spark-csv is actually on maven central. dfdb07b [Felix Cheung] add test eab697b [Felix Cheung] Make custom remote repo configurable Project: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/commit/86a1a199 Tree: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/tree/86a1a199 Diff: http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/diff/86a1a199 Branch: refs/heads/master Commit: 86a1a199453b3c347e2b4e731463fea1bc5940bd Parents: fff175f Author: Felix Cheung <[email protected]> Authored: Sun Oct 25 17:01:18 2015 -0700 Committer: Lee moon soo <[email protected]> Committed: Wed Oct 28 13:06:08 2015 +0100 ---------------------------------------------------------------------- .../apache/zeppelin/spark/DepInterpreter.java | 6 +++- .../apache/zeppelin/spark/SparkInterpreter.java | 5 +++- .../org/apache/zeppelin/spark/dep/Booter.java | 5 ++++ .../zeppelin/spark/dep/DependencyContext.java | 31 ++++++++++++++++---- .../zeppelin/spark/dep/DependencyResolver.java | 24 +++++++++++++-- .../zeppelin/spark/DepInterpreterTest.java | 7 +++++ 6 files changed, 68 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java index 16bd0f0..c4614bf 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/DepInterpreter.java @@ -68,6 +68,9 @@ public class DepInterpreter extends Interpreter { DepInterpreter.class.getName(), new InterpreterPropertyBuilder() .add("zeppelin.dep.localrepo", "local-repo", "local repository for dependency loader") + .add("zeppelin.dep.additionalRemoteRepository", + "spark-packages,http://dl.bintray.com/spark-packages/maven,false;";, + "A list of 'id,remote-repository-URL,is-snapshot;' for each remote repository.") .build()); } @@ -146,7 +149,8 @@ public class DepInterpreter extends Interpreter { intp.setContextClassLoader(); intp.initializeSynchronous(); - depc = new DependencyContext(getProperty("zeppelin.dep.localrepo")); + depc = new DependencyContext(getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); completor = new SparkJLineCompletion(intp); intp.interpret("@transient var _binder = new java.util.HashMap[String, Object]()"); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index a2e31fb..77d4986 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -228,7 +228,10 @@ public class SparkInterpreter extends Interpreter { public DependencyResolver getDependencyResolver() { if (dep == null) { - dep = new DependencyResolver(intp, sc, getProperty("zeppelin.dep.localrepo")); + dep = new DependencyResolver(intp, + sc, + getProperty("zeppelin.dep.localrepo"), + getProperty("zeppelin.dep.additionalRemoteRepository")); } return dep; } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java index 0533804..ece39c3 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/Booter.java @@ -67,4 +67,9 @@ public class Booter { public static RemoteRepository newCentralRepository() { return new RemoteRepository("central", "default", "http://repo1.maven.org/maven2/";); } + + public static RemoteRepository newLocalRepository() { + return new RemoteRepository("local", + "default", "file://" + System.getProperty("user.home") + "/.m2/repository"); + } } http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java index f0fd313..0f465f1 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyContext.java @@ -49,13 +49,13 @@ public class DependencyContext { List<File> filesDist = new LinkedList<File>(); private RepositorySystem system = Booter.newRepositorySystem(); private RepositorySystemSession session; - private RemoteRepository mavenCentral = new RemoteRepository("central", - "default", "http://repo1.maven.org/maven2/";); - private RemoteRepository mavenLocal = new RemoteRepository("local", - "default", "file://" + System.getProperty("user.home") + "/.m2/repository"); + private RemoteRepository mavenCentral = Booter.newCentralRepository(); + private RemoteRepository mavenLocal = Booter.newLocalRepository(); + private List<RemoteRepository> additionalRepos = new LinkedList<RemoteRepository>(); - public DependencyContext(String localRepoPath) { + public DependencyContext(String localRepoPath, String additionalRemoteRepository) { session = Booter.newRepositorySystemSession(system, localRepoPath); + addRepoFromProperty(additionalRemoteRepository); } public Dependency load(String lib) { @@ -82,6 +82,24 @@ public class DependencyContext { filesDist = new LinkedList<File>(); } + private void addRepoFromProperty(String listOfRepo) { + if (listOfRepo != null) { + String[] repos = listOfRepo.split(";"); + for (String repo : repos) { + String[] parts = repo.split(","); + if (parts.length == 3) { + String id = parts[0].trim(); + String url = parts[1].trim(); + boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); + if (id.length() > 1 && url.length() > 1) { + RemoteRepository rr = new RemoteRepository(id, "default", url); + rr.setPolicy(isSnapshot, null); + additionalRepos.add(rr); + } + } + } + } + } /** * fetch all artifacts @@ -129,6 +147,9 @@ public class DependencyContext { collectRequest.addRepository(mavenCentral); collectRequest.addRepository(mavenLocal); + for (RemoteRepository repo : additionalRepos) { + collectRequest.addRepository(repo); + } for (Repository repo : repositories) { RemoteRepository rr = new RemoteRepository(repo.getName(), "default", repo.getUrl()); rr.setPolicy(repo.isSnapshot(), null); http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java ---------------------------------------------------------------------- diff --git a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java index 0702948..0a30530 100644 --- a/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java +++ b/spark/src/main/java/org/apache/zeppelin/spark/dep/DependencyResolver.java @@ -83,14 +83,15 @@ public class DependencyResolver { "org.apache.zeppelin:zeppelin-spark", "org.apache.zeppelin:zeppelin-server"}; - public DependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath) { + public DependencyResolver(SparkIMain intp, SparkContext sc, String localRepoPath, + String additionalRemoteRepository) { this.intp = intp; this.global = intp.global(); this.sc = sc; session = Booter.newRepositorySystemSession(system, localRepoPath); repos.add(Booter.newCentralRepository()); // add maven central - repos.add(new RemoteRepository("local", "default", "file://" - + System.getProperty("user.home") + "/.m2/repository")); + repos.add(Booter.newLocalRepository()); + addRepoFromProperty(additionalRemoteRepository); } public void addRepo(String id, String url, boolean snapshot) { @@ -116,6 +117,23 @@ public class DependencyResolver { return null; } + private void addRepoFromProperty(String listOfRepo) { + if (listOfRepo != null) { + String[] repos = listOfRepo.split(";"); + for (String repo : repos) { + String[] parts = repo.split(","); + if (parts.length == 3) { + String id = parts[0].trim(); + String url = parts[1].trim(); + boolean isSnapshot = Boolean.parseBoolean(parts[2].trim()); + if (id.length() > 1 && url.length() > 1) { + addRepo(id, url, isSnapshot); + } + } + } + } + } + private void updateCompilerClassPath(URL[] urls) throws IllegalAccessException, IllegalArgumentException, InvocationTargetException { http://git-wip-us.apache.org/repos/asf/incubator-zeppelin/blob/86a1a199/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java ---------------------------------------------------------------------- diff --git a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java index 6adaa14..efa8fae 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/DepInterpreterTest.java @@ -90,5 +90,12 @@ public class DepInterpreterTest { assertEquals(1, dep.getDependencyContext().getFiles().size()); assertEquals(1, dep.getDependencyContext().getFilesDist().size()); + + // Add a test for the spark-packages repo - default in additionalRemoteRepository + ret = dep.interpret("z.load(\"amplab:spark-indexedrdd:0.3\")", context); + assertEquals(Code.SUCCESS, ret.code()); + + // Reset at the end of the test + dep.getDependencyContext().reset(); } }
