Repository: zeppelin Updated Branches: refs/heads/master d20dbce30 -> 2a2a2e82b
[ZEPPELIN-1075] Merge NoteInterpreterLoader into InterpreterFactory ### What is this PR for? Removing redundant codes between `NoteInterpreterLoader` and `InterpreterFactory`, reducing the cost to add new features, and making refactoring on `InterpreterFactory` easy ### What type of PR is it? [Refactoring] ### Todos * [x] - Remove `NoteInterpreterLoader` and move the functionality into `InterpreterFactory` ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-1075 ### How should this be tested? Must pass all tests ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jongyoul Lee <[email protected]> Closes #1102 from jongyoul/ZEPPELIN-1075 and squashes the following commits: d9816f1 [Jongyoul Lee] Fixed related codes a2d8104 [Jongyoul Lee] Fixed some style and removed unused codes 28bf520 [Jongyoul Lee] Fixed style 600de98 [Jongyoul Lee] Removed NoteInterpreterLoader Fixed some tests 536059b [Jongyoul Lee] Duplicated all method in NoteInterpreterLoader to InterpreterFactory Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/2a2a2e82 Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/2a2a2e82 Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/2a2a2e82 Branch: refs/heads/master Commit: 2a2a2e82b9de92d70897ce8e81ae1498949598fd Parents: d20dbce Author: Jongyoul Lee <[email protected]> Authored: Thu Jun 30 23:46:56 2016 +0900 Committer: Jongyoul Lee <[email protected]> Committed: Sat Jul 2 19:37:46 2016 +0900 ---------------------------------------------------------------------- .../apache/zeppelin/socket/NotebookServer.java | 20 +- .../zeppelin/rest/InterpreterRestApiTest.java | 2 +- .../zeppelin/socket/NotebookServerTest.java | 2 +- .../interpreter/InterpreterFactory.java | 320 +++++++++++++------ .../java/org/apache/zeppelin/notebook/Note.java | 40 ++- .../notebook/NoteInterpreterLoader.java | 207 ------------ .../org/apache/zeppelin/notebook/Notebook.java | 23 +- .../org/apache/zeppelin/notebook/Paragraph.java | 24 +- .../notebook/NoteInterpreterLoaderTest.java | 96 +++--- .../org/apache/zeppelin/notebook/NoteTest.java | 47 +-- .../apache/zeppelin/notebook/NotebookTest.java | 53 ++- .../apache/zeppelin/notebook/ParagraphTest.java | 11 +- .../notebook/repo/VFSNotebookRepoTest.java | 2 +- .../zeppelin/search/LuceneSearchTest.java | 14 +- 14 files changed, 376 insertions(+), 485 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index f4cf9d9..42edb08 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -307,7 +307,7 @@ public class NotebookServer extends WebSocketServlet implements Notebook notebook = notebook(); List<Note> notes = notebook.getAllNotes(); for (Note note : notes) { - List<String> ids = note.getNoteReplLoader().getInterpreters(); + List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId()); for (String id : ids) { if (id.equals(interpreterGroupId)) { broadcast(note.id(), m); @@ -750,8 +750,8 @@ public class NotebookServer extends WebSocketServlet implements // propagate change to (Remote) AngularObjectRegistry Note note = notebook.getNote(noteId); if (note != null) { - List<InterpreterSetting> settings = note.getNoteReplLoader() - .getInterpreterSettings(); + List<InterpreterSetting> settings = notebook.getInterpreterFactory() + .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup(note.id()) == null) { continue; @@ -791,8 +791,8 @@ public class NotebookServer extends WebSocketServlet implements if (global) { // broadcast change to all web session that uses related // interpreter. for (Note n : notebook.getAllNotes()) { - List<InterpreterSetting> settings = note.getNoteReplLoader() - .getInterpreterSettings(); + List<InterpreterSetting> settings = notebook.getInterpreterFactory() + .getInterpreterSettings(note.getId()); for (InterpreterSetting setting : settings) { if (setting.getInterpreterGroup(n.id()) == null) { continue; @@ -1239,8 +1239,8 @@ public class NotebookServer extends WebSocketServlet implements } private void sendAllAngularObjects(Note note, NotebookSocket conn) throws IOException { - List<InterpreterSetting> settings = note.getNoteReplLoader() - .getInterpreterSettings(); + List<InterpreterSetting> settings = + notebook().getInterpreterFactory().getInterpreterSettings(note.getId()); if (settings == null || settings.size() == 0) { return; } @@ -1279,8 +1279,8 @@ public class NotebookServer extends WebSocketServlet implements continue; } - List<InterpreterSetting> intpSettings = note.getNoteReplLoader() - .getInterpreterSettings(); + List<InterpreterSetting> intpSettings = notebook.getInterpreterFactory() + .getInterpreterSettings(note.getId()); if (intpSettings.isEmpty()) continue; for (InterpreterSetting setting : intpSettings) { @@ -1306,7 +1306,7 @@ public class NotebookServer extends WebSocketServlet implements continue; } - List<String> ids = note.getNoteReplLoader().getInterpreters(); + List<String> ids = notebook.getInterpreterFactory().getInterpreters(note.getId()); for (String id : ids) { if (id.equals(interpreterGroupId)) { broadcast( http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java index f03d87b..4d7caf1 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -159,7 +159,7 @@ public class InterpreterRestApiTest extends AbstractTestRestApi { // restart interpreter - for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { + for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { // Call Restart Interpreter REST API PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java index ee2c7c6..bc13113 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/socket/NotebookServerTest.java @@ -93,7 +93,7 @@ public class NotebookServerTest extends AbstractTestRestApi { // get reference to interpreterGroup InterpreterGroup interpreterGroup = null; - List<InterpreterSetting> settings = note1.getNoteReplLoader().getInterpreterSettings(); + List<InterpreterSetting> settings = notebook.getInterpreterFactory().getInterpreterSettings(note1.getId()); for (InterpreterSetting setting : settings) { if (setting.getName().equals("md")) { interpreterGroup = setting.getInterpreterGroup("sharedProcess"); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index ca9e471..5595c14 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -34,7 +34,6 @@ import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; -import org.apache.zeppelin.notebook.NoteInterpreterLoader; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; import org.slf4j.Logger; @@ -60,27 +59,28 @@ import java.util.*; * Manage interpreters. */ public class InterpreterFactory implements InterpreterGroupFactory { - Logger logger = LoggerFactory.getLogger(InterpreterFactory.class); + private static Logger logger = LoggerFactory.getLogger(InterpreterFactory.class); - private Map<String, URLClassLoader> cleanCl = Collections - .synchronizedMap(new HashMap<String, URLClassLoader>()); + private static final String SHARED_SESSION = "shared_session"; + + private Map<String, URLClassLoader> cleanCl = + Collections.synchronizedMap(new HashMap<String, URLClassLoader>()); private ZeppelinConfiguration conf; @Deprecated - String[] interpreterClassList; - String[] interpreterGroupOrderList; + private String[] interpreterClassList; + private String[] interpreterGroupOrderList; - private Map<String, InterpreterSetting> interpreterSettings = - new HashMap<String, InterpreterSetting>(); + private Map<String, InterpreterSetting> interpreterSettings = new HashMap<>(); - private Map<String, List<String>> interpreterBindings = new HashMap<String, List<String>>(); + private Map<String, List<String>> interpreterBindings = new HashMap<>(); private List<RemoteRepository> interpreterRepositories; private Gson gson; private InterpreterOption defaultOption; - AngularObjectRegistryListener angularObjectRegistryListener; + private AngularObjectRegistryListener angularObjectRegistryListener; private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private DependencyResolver depResolver; @@ -91,7 +91,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, - remoteInterpreterProcessListener, depResolver); + remoteInterpreterProcessListener, depResolver); } @@ -230,7 +230,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { } private void registerInterpreterFromResource(ClassLoader cl, String interpreterDir, - String interpreterJson) + String interpreterJson) throws MalformedURLException { URL[] urls = recursiveBuildLibList(new File(interpreterDir)); ClassLoader tempClassLoader = new URLClassLoader(urls, cl); @@ -246,7 +246,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { } private void registerInterpreterFromPath(String interpreterDir, - String interpreterJson) throws IOException { + String interpreterJson) throws IOException { Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); if (Files.exists(interpreterJsonPath)) { @@ -269,11 +269,11 @@ public class InterpreterFactory implements InterpreterGroupFactory { } private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters, - String absolutePath) { + String absolutePath) { for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { String className = registeredInterpreter.getClassName(); if (validateRegisterInterpreter(registeredInterpreter) && - null == Interpreter.findRegisteredInterpreterByClassName(className)) { + null == Interpreter.findRegisteredInterpreterByClassName(className)) { registeredInterpreter.setPath(absolutePath); Interpreter.register(registeredInterpreter); logger.debug("Registered. key: {}, className: {}, path: {}", @@ -285,7 +285,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { private boolean validateRegisterInterpreter(RegisteredInterpreter registeredInterpreter) { return null != registeredInterpreter.getGroup() && null != registeredInterpreter.getName() && - null != registeredInterpreter.getClassName(); + null != registeredInterpreter.getClassName(); } private void loadFromFile() throws IOException { @@ -323,14 +323,9 @@ public class InterpreterFactory implements InterpreterGroupFactory { // previously created setting should turn this feature on here. setting.getOption().setRemote(true); - InterpreterSetting intpSetting = new InterpreterSetting( - setting.id(), - setting.getName(), - setting.getGroup(), - setting.getInterpreterInfos(), - setting.getProperties(), - setting.getDependencies(), - setting.getOption()); + InterpreterSetting intpSetting = new InterpreterSetting(setting.id(), setting.getName(), + setting.getGroup(), setting.getInterpreterInfos(), setting.getProperties(), + setting.getDependencies(), setting.getOption()); intpSetting.setInterpreterGroupFactory(this); interpreterSettings.put(k, intpSetting); @@ -358,18 +353,14 @@ public class InterpreterFactory implements InterpreterGroupFactory { // load dependencies List<Dependency> deps = intSetting.getDependencies(); if (deps != null) { - for (Dependency d: deps) { + for (Dependency d : deps) { File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); if (d.getExclusions() != null) { - depResolver.load( - d.getGroupArtifactVersion(), - d.getExclusions(), + depResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), new File(destDir, intSetting.id())); } else { - depResolver.load( - d.getGroupArtifactVersion(), - new File(destDir, intSetting.id())); + depResolver.load(d.getGroupArtifactVersion(), new File(destDir, intSetting.id())); } } } @@ -399,29 +390,19 @@ public class InterpreterFactory implements InterpreterGroupFactory { fos.close(); } - private RegisteredInterpreter getRegisteredReplInfoFromClassName(String clsName) { - Set<String> keys = Interpreter.registeredInterpreters.keySet(); - for (String intName : keys) { - RegisteredInterpreter info = Interpreter.registeredInterpreters.get(intName); - if (clsName.equals(info.getClassName())) { - return info; - } - } - return null; - } - /** * Return ordered interpreter setting list. * The list does not contain more than one setting from the same interpreter class. * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name + * * @return */ public List<String> getDefaultInterpreterSettingList() { // this list will contain default interpreter setting list - List<String> defaultSettings = new LinkedList<String>(); + List<String> defaultSettings = new LinkedList<>(); // to ignore the same interpreter group - Map<String, Boolean> interpreterGroupCheck = new HashMap<String, Boolean>(); + Map<String, Boolean> interpreterGroupCheck = new HashMap<>(); List<InterpreterSetting> sortedSettings = get(); @@ -443,15 +424,14 @@ public class InterpreterFactory implements InterpreterGroupFactory { } /** - * @param name user defined name - * @param groupName interpreter group name to instantiate + * @param name user defined name + * @param groupName interpreter group name to instantiate * @param properties * @return * @throws InterpreterException * @throws IOException */ - public InterpreterSetting add(String name, String groupName, - List<Dependency> dependencies, + public InterpreterSetting add(String name, String groupName, List<Dependency> dependencies, InterpreterOption option, Properties properties) throws InterpreterException, IOException, RepositoryException { synchronized (interpreterSettings) { @@ -471,13 +451,8 @@ public class InterpreterFactory implements InterpreterGroupFactory { } } - InterpreterSetting intpSetting = new InterpreterSetting( - name, - groupName, - interpreterInfos, - properties, - dependencies, - option); + InterpreterSetting intpSetting = new InterpreterSetting(name, groupName, interpreterInfos, + properties, dependencies, option); if (dependencies.size() > 0) { loadInterpreterDependencies(intpSetting); @@ -502,15 +477,10 @@ public class InterpreterFactory implements InterpreterGroupFactory { InterpreterGroup interpreterGroup = new InterpreterGroup(id); if (option.isRemote()) { - angularObjectRegistry = new RemoteAngularObjectRegistry( - id, - angularObjectRegistryListener, - interpreterGroup - ); + angularObjectRegistry = new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, + interpreterGroup); } else { - angularObjectRegistry = new AngularObjectRegistry( - id, - angularObjectRegistryListener); + angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener); // TODO(moon) : create distributed resource pool for local interpreters and set } @@ -519,8 +489,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { return interpreterGroup; } - public void removeInterpretersForNote(InterpreterSetting interpreterSetting, - String noteId) { + public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String noteId) { if (interpreterSetting.getOption().isPerNoteProcess()) { interpreterSetting.closeAndRemoveInterpreterGroup(noteId); } else if (interpreterSetting.getOption().isPerNoteSession()) { @@ -538,9 +507,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { } } - public void createInterpretersForNote( - InterpreterSetting interpreterSetting, - String noteId, + public void createInterpretersForNote(InterpreterSetting interpreterSetting, String noteId, String key) { InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId); String groupName = interpreterSetting.getGroup(); @@ -557,10 +524,8 @@ public class InterpreterFactory implements InterpreterGroupFactory { // in ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT msec. However, if termination of the process and // removal from interpreter group take too long, throw an error. long minTimeout = 10L * 1000 * 1000000; // 10 sec - long interpreterRemovalWaitTimeout = - Math.max( - minTimeout, - conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2); + long interpreterRemovalWaitTimeout = Math.max(minTimeout, + conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT) * 1000000L * 2); while (interpreterGroup.containsKey(key)) { if (System.nanoTime() - interpreterRemovalWaitStart > interpreterRemovalWaitTimeout) { throw new InterpreterException("Can not create interpreter"); @@ -582,21 +547,16 @@ public class InterpreterFactory implements InterpreterGroupFactory { Interpreter intp; if (option.isRemote()) { - intp = createRemoteRepl(info.getPath(), - key, - info.getClassName(), - properties, + intp = createRemoteRepl(info.getPath(), key, info.getClassName(), properties, interpreterSetting.id()); } else { - intp = createRepl(info.getPath(), - info.getClassName(), - properties); + intp = createRepl(info.getPath(), info.getClassName(), properties); } synchronized (interpreterGroup) { List<Interpreter> interpreters = interpreterGroup.get(key); if (interpreters == null) { - interpreters = new LinkedList<Interpreter>(); + interpreters = new LinkedList<>(); interpreterGroup.put(key, interpreters); } if (info.isDefaultInterpreter()) { @@ -612,7 +572,6 @@ public class InterpreterFactory implements InterpreterGroupFactory { } - public void remove(String id) throws IOException { synchronized (interpreterSettings) { if (interpreterSettings.containsKey(id)) { @@ -639,11 +598,12 @@ public class InterpreterFactory implements InterpreterGroupFactory { /** * Get interpreter settings + * * @return */ public List<InterpreterSetting> get() { synchronized (interpreterSettings) { - List<InterpreterSetting> orderedSettings = new LinkedList<InterpreterSetting>(); + List<InterpreterSetting> orderedSettings = new LinkedList<>(); Map<String, List<InterpreterSetting>> groupNameInterpreterSettingMap = new HashMap<>(); for (InterpreterSetting interpreterSetting : interpreterSettings.values()) { @@ -692,9 +652,9 @@ public class InterpreterFactory implements InterpreterGroupFactory { } } - public void putNoteInterpreterSettingBinding(String noteId, - List<String> settingList) throws IOException { - List<String> unBindedSettings = new LinkedList<String>(); + private void putNoteInterpreterSettingBinding(String noteId, List<String> settingList) + throws IOException { + List<String> unBindedSettings = new LinkedList<>(); synchronized (interpreterSettings) { List<String> oldSettings = interpreterBindings.get(noteId); @@ -725,8 +685,8 @@ public class InterpreterFactory implements InterpreterGroupFactory { } } - public List<String> getNoteInterpreterSettingBinding(String noteId) { - LinkedList<String> bindings = new LinkedList<String>(); + private List<String> getNoteInterpreterSettingBinding(String noteId) { + LinkedList<String> bindings = new LinkedList<>(); synchronized (interpreterSettings) { List<String> settingIds = interpreterBindings.get(noteId); if (settingIds != null) { @@ -738,14 +698,13 @@ public class InterpreterFactory implements InterpreterGroupFactory { /** * Change interpreter property and restart + * * @param id * @param option * @param properties * @throws IOException */ - public void setPropertyAndRestart(String id, - InterpreterOption option, - Properties properties, + public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties, List<Dependency> dependencies) throws IOException, RepositoryException { synchronized (interpreterSettings) { InterpreterSetting intpsetting = interpreterSettings.get(id); @@ -806,7 +765,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { } public void close() { - List<Thread> closeThreads = new LinkedList<Thread>(); + List<Thread> closeThreads = new LinkedList<>(); synchronized (interpreterSettings) { Collection<InterpreterSetting> intpsettings = interpreterSettings.values(); for (final InterpreterSetting intpsetting : intpsettings) { @@ -842,7 +801,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { URLClassLoader ccl = cleanCl.get(dirName); if (ccl == null) { // classloader fallback - ccl = URLClassLoader.newInstance(new URL[] {}, oldcl); + ccl = URLClassLoader.newInstance(new URL[]{}, oldcl); } boolean separateCL = true; @@ -852,13 +811,13 @@ public class InterpreterFactory implements InterpreterGroupFactory { separateCL = false; } } catch (Exception e) { - logger.error("exception checking server classloader driver" , e); + logger.error("exception checking server classloader driver", e); } URLClassLoader cl; if (separateCL == true) { - cl = URLClassLoader.newInstance(new URL[] {}, ccl); + cl = URLClassLoader.newInstance(new URL[]{}, ccl); } else { cl = ccl; } @@ -866,7 +825,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { Class<Interpreter> replClass = (Class<Interpreter>) cl.loadClass(className); Constructor<Interpreter> constructor = - replClass.getConstructor(new Class[] {Properties.class}); + replClass.getConstructor(new Class[]{Properties.class}); Interpreter repl = constructor.newInstance(property); repl.setClassloaderUrls(ccl.getURLs()); LazyOpenInterpreter intp = new LazyOpenInterpreter( @@ -900,15 +859,14 @@ public class InterpreterFactory implements InterpreterGroupFactory { updatePropertiesFromRegisteredInterpreter(property, className); - LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter( - property, noteId, className, conf.getInterpreterRemoteRunnerPath(), - interpreterPath, localRepoPath, connectTimeout, - maxPoolSize, remoteInterpreterProcessListener)); + LazyOpenInterpreter intp = new LazyOpenInterpreter(new RemoteInterpreter(property, noteId, + className, conf.getInterpreterRemoteRunnerPath(), interpreterPath, localRepoPath, + connectTimeout, maxPoolSize, remoteInterpreterProcessListener)); return intp; } private Properties updatePropertiesFromRegisteredInterpreter(Properties properties, - String className) { + String className) { RegisteredInterpreter registeredInterpreter = Interpreter.findRegisteredInterpreterByClassName( className); if (null != registeredInterpreter) { @@ -923,10 +881,168 @@ public class InterpreterFactory implements InterpreterGroupFactory { return properties; } + /** + * map interpreter ids into noteId + * + * @param noteId note id + * @param ids InterpreterSetting id list + * @throws IOException + */ + public void setInterpreters(String noteId, List<String> ids) throws IOException { + putNoteInterpreterSettingBinding(noteId, ids); + } + + public List<String> getInterpreters(String noteId) { + return getNoteInterpreterSettingBinding(noteId); + } + + public List<InterpreterSetting> getInterpreterSettings(String noteId) { + List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId); + LinkedList<InterpreterSetting> settings = new LinkedList<>(); + synchronized (interpreterSettingIds) { + for (String id : interpreterSettingIds) { + InterpreterSetting setting = get(id); + if (setting == null) { + // interpreter setting is removed from factory. remove id from here, too + interpreterSettingIds.remove(id); + } else { + settings.add(setting); + } + } + } + return settings; + } + + public void closeNote(String noteId) { + // close interpreters in this note session + List<InterpreterSetting> settings = getInterpreterSettings(noteId); + if (settings == null || settings.size() == 0) { + return; + } + + logger.info("closeNote: {}", noteId); + for (InterpreterSetting setting : settings) { + removeInterpretersForNote(setting, noteId); + } + } + + private String getInterpreterInstanceKey(String noteId, InterpreterSetting setting) { + if (setting.getOption().isExistingProcess()) { + return Constants.EXISTING_PROCESS; + } else if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) { + return noteId; + } else { + return SHARED_SESSION; + } + } + + private List<Interpreter> createOrGetInterpreterList(String noteId, InterpreterSetting setting) { + InterpreterGroup interpreterGroup = setting.getInterpreterGroup(noteId); + synchronized (interpreterGroup) { + String key = getInterpreterInstanceKey(noteId, setting); + if (!interpreterGroup.containsKey(key)) { + createInterpretersForNote(setting, noteId, key); + } + return interpreterGroup.get(getInterpreterInstanceKey(noteId, setting)); + } + } + + private InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) { + if (settings == null || settings.isEmpty()) { + return null; + } + return settings.get(0); + } + + public InterpreterSetting getDefaultInterpreterSetting(String noteId) { + return getDefaultInterpreterSetting(getInterpreterSettings(noteId)); + } + + public Interpreter getInterpreter(String noteId, String replName) { + List<InterpreterSetting> settings = getInterpreterSettings(noteId); + + if (settings == null || settings.size() == 0) { + return null; + } + + if (replName == null || replName.trim().length() == 0) { + // get default settings (first available) + // TODO(jl): Fix it in case of returning null + InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings); + return createOrGetInterpreterList(noteId, defaultSettings).get(0); + } + + if (Interpreter.registeredInterpreters == null) { + return null; + } + + String[] replNameSplit = replName.split("\\."); + String group = null; + String name = null; + if (replNameSplit.length == 2) { + group = replNameSplit[0]; + name = replNameSplit[1]; + + Interpreter.RegisteredInterpreter registeredInterpreter = Interpreter.registeredInterpreters + .get(group + "." + name); + if (registeredInterpreter == null + || registeredInterpreter.getClassName() == null) { + throw new InterpreterException(replName + " interpreter not found"); + } + String interpreterClassName = registeredInterpreter.getClassName(); + + for (InterpreterSetting setting : settings) { + if (registeredInterpreter.getGroup().equals(setting.getGroup())) { + List<Interpreter> intpGroup = createOrGetInterpreterList(noteId, setting); + for (Interpreter interpreter : intpGroup) { + if (interpreterClassName.equals(interpreter.getClassName())) { + return interpreter; + } + } + } + } + throw new InterpreterException(replName + " interpreter not found"); + } else { + // first assume replName is 'name' of interpreter. ('groupName' is ommitted) + // search 'name' from first (default) interpreter group + InterpreterSetting defaultSetting = getDefaultInterpreterSetting(settings); + Interpreter.RegisteredInterpreter registeredInterpreter = + Interpreter.registeredInterpreters.get(defaultSetting.getGroup() + "." + replName); + if (registeredInterpreter != null) { + List<Interpreter> interpreters = createOrGetInterpreterList(noteId, defaultSetting); + for (Interpreter interpreter : interpreters) { + + RegisteredInterpreter intp = + Interpreter.findRegisteredInterpreterByClassName(interpreter.getClassName()); + if (intp == null) { + continue; + } + + if (intp.getName().equals(replName)) { + return interpreter; + } + } + + throw new InterpreterException( + defaultSetting.getGroup() + "." + replName + " interpreter not found"); + } + + // next, assume replName is 'group' of interpreter ('name' is ommitted) + // search interpreter group and return first interpreter. + for (InterpreterSetting setting : settings) { + if (setting.getGroup().equals(replName)) { + List<Interpreter> interpreters = createOrGetInterpreterList(noteId, setting); + return interpreters.get(0); + } + } + } + + return null; + } private URL[] recursiveBuildLibList(File path) throws MalformedURLException { URL[] urls = new URL[0]; - if (path == null || path.exists() == false) { + if (path == null || !path.exists()) { return urls; } else if (path.getName().startsWith(".")) { return urls; @@ -939,7 +1055,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { } return urls; } else { - return new URL[] {path.toURI().toURL()}; + return new URL[]{path.toURI().toURL()}; } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 1f4a508..e57ed9b 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -73,7 +73,7 @@ public class Note implements Serializable, JobListener { @SuppressWarnings("rawtypes") Map<String, List<AngularObject>> angularObjects = new HashMap<>(); - private transient NoteInterpreterLoader replLoader; + private transient InterpreterFactory factory; private transient JobListenerFactory jobListenerFactory; private transient NotebookRepo repo; private transient SearchService index; @@ -97,10 +97,10 @@ public class Note implements Serializable, JobListener { public Note() {} - public Note(NotebookRepo repo, NoteInterpreterLoader replLoader, + public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials) { this.repo = repo; - this.replLoader = replLoader; + this.factory = factory; this.jobListenerFactory = jlFactory; this.index = noteIndex; this.credentials = credentials; @@ -112,8 +112,8 @@ public class Note implements Serializable, JobListener { } private String getDefaultInterpreterName() { - Optional<InterpreterSetting> settingOptional = replLoader.getDefaultInterpreterSetting(); - return settingOptional.isPresent() ? settingOptional.get().getGroup() : StringUtils.EMPTY; + InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId()); + return null != setting ? setting.getGroup() : StringUtils.EMPTY; } void putDefaultReplName() { @@ -154,12 +154,8 @@ public class Note implements Serializable, JobListener { this.name = name; } - public NoteInterpreterLoader getNoteReplLoader() { - return replLoader; - } - - public void setReplLoader(NoteInterpreterLoader replLoader) { - this.replLoader = replLoader; + public void setInterpreterFactory(InterpreterFactory factory) { + this.factory = factory; } public JobListenerFactory getJobListenerFactory() { @@ -201,7 +197,7 @@ public class Note implements Serializable, JobListener { */ public Paragraph addParagraph() { - Paragraph p = new Paragraph(this, this, replLoader); + Paragraph p = new Paragraph(this, this, factory); addLastReplNameIfEmptyText(p); synchronized (paragraphs) { paragraphs.add(p); @@ -217,7 +213,7 @@ public class Note implements Serializable, JobListener { public void addCloneParagraph(Paragraph srcParagraph) { // Keep paragraph original ID - final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, replLoader); + final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory); Map<String, Object> config = new HashMap<>(srcParagraph.getConfig()); Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams()); @@ -251,7 +247,7 @@ public class Note implements Serializable, JobListener { * @param index */ public Paragraph insertParagraph(int index) { - Paragraph p = new Paragraph(this, this, replLoader); + Paragraph p = new Paragraph(this, this, factory); addLastReplNameIfEmptyText(p); synchronized (paragraphs) { paragraphs.add(index, p); @@ -435,9 +431,9 @@ public class Note implements Serializable, JobListener { AuthenticationInfo authenticationInfo = new AuthenticationInfo(); authenticationInfo.setUser(cronExecutingUser); p.setAuthenticationInfo(authenticationInfo); - p.setNoteReplLoader(replLoader); + p.setInterpreterFactory(factory); p.setListener(jobListenerFactory.getParagraphJobListener(this)); - Interpreter intp = replLoader.get(p.getRequiredReplName()); + Interpreter intp = factory.getInterpreter(getId(), p.getRequiredReplName()); intp.getScheduler().submit(p); } } @@ -450,13 +446,13 @@ public class Note implements Serializable, JobListener { */ public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); - p.setNoteReplLoader(replLoader); + p.setInterpreterFactory(factory); p.setListener(jobListenerFactory.getParagraphJobListener(this)); String requiredReplName = p.getRequiredReplName(); - Interpreter intp = replLoader.get(requiredReplName); + Interpreter intp = factory.getInterpreter(getId(), requiredReplName); if (intp == null) { // TODO(jongyoul): Make "%jdbc" configurable from JdbcInterpreter - if (conf.getUseJdbcAlias() && null != (intp = replLoader.get("jdbc"))) { + if (conf.getUseJdbcAlias() && null != (intp = factory.getInterpreter(getId(), "jdbc"))) { String pText = p.getText().replaceFirst(requiredReplName, "jdbc(" + requiredReplName + ")"); logger.debug("New paragraph: {}", pText); p.setEffectiveText(pText); @@ -487,7 +483,7 @@ public class Note implements Serializable, JobListener { public List<InterpreterCompletion> completion(String paragraphId, String buffer, int cursor) { Paragraph p = getParagraph(paragraphId); - p.setNoteReplLoader(replLoader); + p.setInterpreterFactory(factory); p.setListener(jobListenerFactory.getParagraphJobListener(this)); List completion = p.completion(buffer, cursor); @@ -503,7 +499,7 @@ public class Note implements Serializable, JobListener { private void snapshotAngularObjectRegistry() { angularObjects = new HashMap<>(); - List<InterpreterSetting> settings = replLoader.getInterpreterSettings(); + List<InterpreterSetting> settings = factory.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } @@ -518,7 +514,7 @@ public class Note implements Serializable, JobListener { private void removeAllAngularObjectInParagraph(String paragraphId) { angularObjects = new HashMap<String, List<AngularObject>>(); - List<InterpreterSetting> settings = replLoader.getInterpreterSettings(); + List<InterpreterSetting> settings = factory.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java deleted file mode 100644 index 3c432cb..0000000 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteInterpreterLoader.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.zeppelin.notebook; - -import com.google.common.base.Optional; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; - -import org.apache.zeppelin.interpreter.Constants; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; -import org.apache.zeppelin.interpreter.InterpreterException; -import org.apache.zeppelin.interpreter.InterpreterFactory; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterSetting; - -/** - * Interpreter loader per note. - */ -public class NoteInterpreterLoader { - private transient InterpreterFactory factory; - private static String SHARED_SESSION = "shared_session"; - String noteId; - - public NoteInterpreterLoader(InterpreterFactory factory) { - this.factory = factory; - } - - public void setNoteId(String noteId) { - this.noteId = noteId; - } - - /** - * set interpreter ids - * @param ids InterpreterSetting id list - * @throws IOException - */ - public void setInterpreters(List<String> ids) throws IOException { - factory.putNoteInterpreterSettingBinding(noteId, ids); - } - - public List<String> getInterpreters() { - return factory.getNoteInterpreterSettingBinding(noteId); - } - - public List<InterpreterSetting> getInterpreterSettings() { - List<String> interpreterSettingIds = factory.getNoteInterpreterSettingBinding(noteId); - LinkedList<InterpreterSetting> settings = new LinkedList<InterpreterSetting>(); - synchronized (interpreterSettingIds) { - for (String id : interpreterSettingIds) { - InterpreterSetting setting = factory.get(id); - if (setting == null) { - // interpreter setting is removed from factory. remove id from here, too - interpreterSettingIds.remove(id); - } else { - settings.add(setting); - } - } - } - return settings; - } - - private String getInterpreterInstanceKey(InterpreterSetting setting) { - if (setting.getOption().isExistingProcess()) { - return Constants.EXISTING_PROCESS; - } else if (setting.getOption().isPerNoteSession() || setting.getOption().isPerNoteProcess()) { - return noteId; - } else { - return SHARED_SESSION; - } - } - - private List<Interpreter> createOrGetInterpreterList(InterpreterSetting setting) { - InterpreterGroup interpreterGroup = - setting.getInterpreterGroup(noteId); - synchronized (interpreterGroup) { - String key = getInterpreterInstanceKey(setting); - if (!interpreterGroup.containsKey(key)) { - factory.createInterpretersForNote(setting, noteId, key); - } - return interpreterGroup.get(getInterpreterInstanceKey(setting)); - } - } - - public void close() { - // close interpreters in this note session - List<InterpreterSetting> settings = this.getInterpreterSettings(); - if (settings == null || settings.size() == 0) { - return; - } - - System.err.println("close"); - for (InterpreterSetting setting : settings) { - factory.removeInterpretersForNote(setting, noteId); - } - } - - public Interpreter get(String replName) { - List<InterpreterSetting> settings = getInterpreterSettings(); - - if (settings == null || settings.size() == 0) { - return null; - } - - if (replName == null || replName.trim().length() == 0) { - // get default settings (first available) - InterpreterSetting defaultSettings = getDefaultInterpreterSetting(settings).get(); - return createOrGetInterpreterList(defaultSettings).get(0); - } - - if (Interpreter.registeredInterpreters == null) { - return null; - } - - String[] replNameSplit = replName.split("\\."); - String group = null; - String name = null; - if (replNameSplit.length == 2) { - group = replNameSplit[0]; - name = replNameSplit[1]; - - Interpreter.RegisteredInterpreter registeredInterpreter = Interpreter.registeredInterpreters - .get(group + "." + name); - if (registeredInterpreter == null - || registeredInterpreter.getClassName() == null) { - throw new InterpreterException(replName + " interpreter not found"); - } - String interpreterClassName = registeredInterpreter.getClassName(); - - for (InterpreterSetting setting : settings) { - if (registeredInterpreter.getGroup().equals(setting.getGroup())) { - List<Interpreter> intpGroup = createOrGetInterpreterList(setting); - for (Interpreter interpreter : intpGroup) { - if (interpreterClassName.equals(interpreter.getClassName())) { - return interpreter; - } - } - } - } - throw new InterpreterException(replName + " interpreter not found"); - } else { - // first assume replName is 'name' of interpreter. ('groupName' is ommitted) - // search 'name' from first (default) interpreter group - InterpreterSetting defaultSetting = getDefaultInterpreterSetting(settings).get(); - Interpreter.RegisteredInterpreter registeredInterpreter = - Interpreter.registeredInterpreters.get(defaultSetting.getGroup() + "." + replName); - if (registeredInterpreter != null) { - List<Interpreter> interpreters = createOrGetInterpreterList(defaultSetting); - for (Interpreter interpreter : interpreters) { - - RegisteredInterpreter intp = - Interpreter.findRegisteredInterpreterByClassName(interpreter.getClassName()); - if (intp == null) { - continue; - } - - if (intp.getName().equals(replName)) { - return interpreter; - } - } - - throw new InterpreterException( - defaultSetting.getGroup() + "." + replName + " interpreter not found"); - } - - // next, assume replName is 'group' of interpreter ('name' is ommitted) - // search interpreter group and return first interpreter. - for (InterpreterSetting setting : settings) { - if (setting.getGroup().equals(replName)) { - List<Interpreter> interpreters = createOrGetInterpreterList(setting); - return interpreters.get(0); - } - } - } - - return null; - } - - private Optional<InterpreterSetting> - getDefaultInterpreterSetting(List<InterpreterSetting> settings) { - if (settings == null || settings.isEmpty()) { - return Optional.absent(); - } - return Optional.of(settings.get(0)); - } - - Optional<InterpreterSetting> getDefaultInterpreterSetting() { - return getDefaultInterpreterSetting(getInterpreterSettings()); - } -} http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index d590223..3243ba7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -151,9 +151,7 @@ public class Notebook { */ public Note createNote(List<String> interpreterIds, AuthenticationInfo subject) throws IOException { - NoteInterpreterLoader intpLoader = new NoteInterpreterLoader(replFactory); - Note note = new Note(notebookRepo, intpLoader, jobListenerFactory, notebookIndex, credentials); - intpLoader.setNoteId(note.id()); + Note note = new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials); synchronized (notes) { notes.put(note.id(), note); } @@ -258,7 +256,7 @@ public class Notebook { List<String> interpreterSettingIds) throws IOException { Note note = getNote(id); if (note != null) { - note.getNoteReplLoader().setInterpreters(interpreterSettingIds); + replFactory.setInterpreters(note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } @@ -267,7 +265,7 @@ public class Notebook { public List<String> getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { - return note.getNoteReplLoader().getInterpreters(); + return getInterpreterFactory().getInterpreters(note.getId()); } else { return new LinkedList<String>(); } @@ -276,7 +274,7 @@ public class Notebook { public List<InterpreterSetting> getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { - return note.getNoteReplLoader().getInterpreterSettings(); + return replFactory.getInterpreterSettings(note.getId()); } else { return new LinkedList<InterpreterSetting>(); } @@ -348,9 +346,7 @@ public class Notebook { note.setIndex(this.notebookIndex); note.setCredentials(this.credentials); - NoteInterpreterLoader replLoader = new NoteInterpreterLoader(replFactory); - note.setReplLoader(replLoader); - replLoader.setNoteId(note.id()); + note.setInterpreterFactory(replFactory); note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); @@ -607,9 +603,9 @@ public class Notebook { // set interpreter bind type String interpreterGroupName = null; - if (note.getNoteReplLoader().getInterpreterSettings() != null && - note.getNoteReplLoader().getInterpreterSettings().size() >= 1) { - interpreterGroupName = note.getNoteReplLoader().getInterpreterSettings().get(0).getGroup(); + if (replFactory.getInterpreterSettings(note.getId()) != null && + replFactory.getInterpreterSettings(note.getId()).size() >= 1) { + interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getGroup(); } // not update and not running -> pass @@ -659,7 +655,8 @@ public class Notebook { logger.error(e.getMessage(), e); } if (releaseResource) { - for (InterpreterSetting setting : note.getNoteReplLoader().getInterpreterSettings()) { + for (InterpreterSetting setting : + notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { notebook.getInterpreterFactory().restart(setting.id()); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index bc53887..df4765d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -49,7 +49,7 @@ import com.google.common.annotations.VisibleForTesting; public class Paragraph extends Job implements Serializable, Cloneable { private static final long serialVersionUID = -6328572073497992016L; - private transient NoteInterpreterLoader replLoader; + private transient InterpreterFactory factory; private transient Note note; private transient AuthenticationInfo authenticationInfo; private transient String effectiveText; @@ -69,10 +69,10 @@ public class Paragraph extends Job implements Serializable, Cloneable { } public Paragraph(String paragraphId, Note note, JobListener listener, - NoteInterpreterLoader replLoader) { + InterpreterFactory factory) { super(paragraphId, generateId(), listener); this.note = note; - this.replLoader = replLoader; + this.factory = factory; title = null; text = null; authenticationInfo = null; @@ -82,10 +82,10 @@ public class Paragraph extends Job implements Serializable, Cloneable { config = new HashMap<String, Object>(); } - public Paragraph(Note note, JobListener listener, NoteInterpreterLoader replLoader) { + public Paragraph(Note note, JobListener listener, InterpreterFactory factory) { super(generateId(), listener); this.note = note; - this.replLoader = replLoader; + this.factory = factory; title = null; text = null; authenticationInfo = null; @@ -194,12 +194,8 @@ public class Paragraph extends Job implements Serializable, Cloneable { return text.substring(magic.length() + 1).trim(); } - public NoteInterpreterLoader getNoteReplLoader() { - return replLoader; - } - public Interpreter getRepl(String name) { - return replLoader.get(name); + return factory.getInterpreter(note.getId(), name); } public Interpreter getCurrentRepl() { @@ -221,8 +217,8 @@ public class Paragraph extends Job implements Serializable, Cloneable { return completion; } - public void setNoteReplLoader(NoteInterpreterLoader repls) { - this.replLoader = repls; + public void setInterpreterFactory(InterpreterFactory factory) { + this.factory = factory; } public InterpreterResult getResult() { @@ -336,8 +332,8 @@ public class Paragraph extends Job implements Serializable, Cloneable { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; - if (!getNoteReplLoader().getInterpreterSettings().isEmpty()) { - InterpreterSetting intpGroup = getNoteReplLoader().getInterpreterSettings().get(0); + if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { + InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(note.id()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(note.id()).getResourcePool(); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java index 147d431..7d9071c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteInterpreterLoaderTest.java @@ -71,102 +71,92 @@ public class NoteInterpreterLoaderTest { @Test public void testGetInterpreter() throws IOException { - NoteInterpreterLoader loader = new NoteInterpreterLoader(factory); - loader.setNoteId("note"); - loader.setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters("note", factory.getDefaultInterpreterSettingList()); // when there're no interpreter selection directive - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", loader.get(null).getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", loader.get("").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", loader.get(" ").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", null).getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", " ").getClassName()); // when group name is omitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", loader.get("mock11").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "mock11").getClassName()); // when 'name' is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", loader.get("group1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", loader.get("group2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2").getClassName()); // when nothing is ommitted - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", loader.get("group1.mock1").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", loader.get("group1.mock11").getClassName()); - assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", loader.get("group2.mock2").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter1", factory.getInterpreter("note", "group1.mock1").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter11", factory.getInterpreter("note", "group1.mock11").getClassName()); + assertEquals("org.apache.zeppelin.interpreter.mock.MockInterpreter2", factory.getInterpreter("note", "group2.mock2").getClassName()); - loader.close(); + factory.closeNote("note"); } @Test public void testNoteSession() throws IOException { - NoteInterpreterLoader loaderA = new NoteInterpreterLoader(factory); - loaderA.setNoteId("noteA"); - loaderA.setInterpreters(factory.getDefaultInterpreterSettingList()); - loaderA.getInterpreterSettings().get(0).getOption().setPerNoteSession(true); + factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList()); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteSession(true); - NoteInterpreterLoader loaderB = new NoteInterpreterLoader(factory); - loaderB.setNoteId("noteB"); - loaderB.setInterpreters(factory.getDefaultInterpreterSettingList()); - loaderB.getInterpreterSettings().get(0).getOption().setPerNoteSession(true); + factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList()); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteSession(true); // interpreters are not created before accessing it - assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); - loaderA.get(null).open(); - loaderB.get(null).open(); + factory.getInterpreter("noteA", null).open(); + factory.getInterpreter("noteB", null).open(); assertTrue( - loaderA.get(null).getInterpreterGroup().getId().equals( - loaderB.get(null).getInterpreterGroup().getId())); + factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( + factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNotNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); // when - loaderA.close(); - loaderB.close(); + factory.closeNote("noteA"); + factory.closeNote("noteB"); // interpreters are destroyed after close - assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteA")); - assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("shared_process").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("shared_process").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("shared_process").get("noteB")); } @Test public void testNotePerInterpreterProcess() throws IOException { - NoteInterpreterLoader loaderA = new NoteInterpreterLoader(factory); - loaderA.setNoteId("noteA"); - loaderA.setInterpreters(factory.getDefaultInterpreterSettingList()); - loaderA.getInterpreterSettings().get(0).getOption().setPerNoteProcess(true); + factory.setInterpreters("noteA", factory.getDefaultInterpreterSettingList()); + factory.getInterpreterSettings("noteA").get(0).getOption().setPerNoteProcess(true); - NoteInterpreterLoader loaderB = new NoteInterpreterLoader(factory); - loaderB.setNoteId("noteB"); - loaderB.setInterpreters(factory.getDefaultInterpreterSettingList()); - loaderB.getInterpreterSettings().get(0).getOption().setPerNoteProcess(true); + factory.setInterpreters("noteB", factory.getDefaultInterpreterSettingList()); + factory.getInterpreterSettings("noteB").get(0).getOption().setPerNoteProcess(true); // interpreters are not created before accessing it - assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA")); - assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); - loaderA.get(null).open(); - loaderB.get(null).open(); + factory.getInterpreter("noteA", null).open(); + factory.getInterpreter("noteB", null).open(); // per note interpreter process assertFalse( - loaderA.get(null).getInterpreterGroup().getId().equals( - loaderB.get(null).getInterpreterGroup().getId())); + factory.getInterpreter("noteA", null).getInterpreterGroup().getId().equals( + factory.getInterpreter("noteB", null).getInterpreterGroup().getId())); // interpreters are created after accessing it - assertNotNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA")); - assertNotNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB")); + assertNotNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); + assertNotNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); // when - loaderA.close(); - loaderB.close(); + factory.closeNote("noteA"); + factory.closeNote("noteB"); // interpreters are destroyed after close - assertNull(loaderA.getInterpreterSettings().get(0).getInterpreterGroup("noteA").get("noteA")); - assertNull(loaderB.getInterpreterSettings().get(0).getInterpreterGroup("noteB").get("noteB")); + assertNull(factory.getInterpreterSettings("noteA").get(0).getInterpreterGroup("noteA").get("noteA")); + assertNull(factory.getInterpreterSettings("noteB").get(0).getInterpreterGroup("noteB").get("noteB")); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index 3e323ef..2ca817f 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -21,6 +21,7 @@ import com.google.common.base.Optional; import org.apache.commons.lang.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.Scheduler; @@ -42,9 +43,6 @@ public class NoteTest { NotebookRepo repo; @Mock - NoteInterpreterLoader replLoader; - - @Mock JobListenerFactory jobListenerFactory; @Mock @@ -59,39 +57,42 @@ public class NoteTest { @Mock Scheduler scheduler; + @Mock + InterpreterFactory interpreterFactory; + @Test public void runNormalTest() { - when(replLoader.get("spark")).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("spark"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%spark sc.version"; - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class); verify(scheduler, only()).submit(pCaptor.capture()); - verify(replLoader, only()).get("spark"); + verify(interpreterFactory, only()).getInterpreter(anyString(), eq("spark")); assertEquals("Paragraph text", pText, pCaptor.getValue().getText()); } @Test public void runJdbcTest() { - when(replLoader.get("mysql")).thenReturn(null); - when(replLoader.get("jdbc")).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("mysql"))).thenReturn(null); + when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter); when(interpreter.getScheduler()).thenReturn(scheduler); String pText = "%mysql show databases"; - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); Paragraph p = note.addParagraph(); p.setText(pText); note.run(p.getId()); ArgumentCaptor<Paragraph> pCaptor = ArgumentCaptor.forClass(Paragraph.class); verify(scheduler, only()).submit(pCaptor.capture()); - verify(replLoader, times(2)).get(anyString()); + verify(interpreterFactory, times(2)).getInterpreter(anyString(), anyString()); assertEquals("Change paragraph text", "%jdbc(mysql) show databases", pCaptor.getValue().getEffectiveText()); assertEquals("Change paragraph text", pText, pCaptor.getValue().getText()); @@ -99,10 +100,10 @@ public class NoteTest { @Test public void putDefaultReplNameIfInterpreterSettingAbsent() { - when(replLoader.getDefaultInterpreterSetting()) - .thenReturn(Optional.<InterpreterSetting>absent()); + when(interpreterFactory.getDefaultInterpreterSetting(anyString())) + .thenReturn(null); - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); note.putDefaultReplName(); assertEquals(StringUtils.EMPTY, note.getLastReplName()); @@ -113,10 +114,10 @@ public class NoteTest { public void putDefaultReplNameIfInterpreterSettingPresent() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); when(interpreterSetting.getGroup()).thenReturn("spark"); - when(replLoader.getDefaultInterpreterSetting()) - .thenReturn(Optional.of(interpreterSetting)); + when(interpreterFactory.getDefaultInterpreterSetting(anyString())) + .thenReturn(interpreterSetting); - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); note.putDefaultReplName(); assertEquals("spark", note.getLastReplName()); @@ -127,10 +128,10 @@ public class NoteTest { public void addParagraphWithLastReplName() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); when(interpreterSetting.getGroup()).thenReturn("spark"); - when(replLoader.getDefaultInterpreterSetting()) - .thenReturn(Optional.of(interpreterSetting)); + when(interpreterFactory.getDefaultInterpreterSetting(anyString())) + .thenReturn(interpreterSetting); - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); note.putDefaultReplName(); //set lastReplName Paragraph p = note.addParagraph(); @@ -142,10 +143,10 @@ public class NoteTest { public void insertParagraphWithLastReplName() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); when(interpreterSetting.getGroup()).thenReturn("spark"); - when(replLoader.getDefaultInterpreterSetting()) - .thenReturn(Optional.of(interpreterSetting)); + when(interpreterFactory.getDefaultInterpreterSetting(anyString())) + .thenReturn(interpreterSetting); - Note note = new Note(repo, replLoader, jobListenerFactory, index, credentials); + Note note = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials); note.putDefaultReplName(); //set lastReplName Paragraph p = note.insertParagraph(note.getParagraphs().size()); @@ -156,7 +157,7 @@ public class NoteTest { @Test public void setLastReplName() { String paragraphId = "HelloWorld"; - Note note = Mockito.spy(new Note(repo, replLoader, jobListenerFactory, index, credentials)); + Note note = Mockito.spy(new Note(repo, interpreterFactory, jobListenerFactory, index, credentials)); Paragraph mockParagraph = Mockito.mock(Paragraph.class); when(note.getParagraph(paragraphId)).thenReturn(mockParagraph); when(mockParagraph.getRequiredReplName()).thenReturn("spark"); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 0305b21..13771b7 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -105,7 +105,7 @@ public class NotebookTest implements JobListenerFactory{ @Test public void testSelectingReplImplementation() throws IOException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); // run with defatul repl Paragraph p1 = note.addParagraph(); @@ -203,7 +203,7 @@ public class NotebookTest implements JobListenerFactory{ @Test public void testRunAll() throws IOException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); // p1 Paragraph p1 = note.addParagraph(); @@ -242,7 +242,7 @@ public class NotebookTest implements JobListenerFactory{ public void testSchedule() throws InterruptedException, IOException{ // create a note and a paragraph Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); Map config = new HashMap<String, Object>(); @@ -274,7 +274,7 @@ public class NotebookTest implements JobListenerFactory{ public void testAutoRestartInterpreterAfterSchedule() throws InterruptedException, IOException{ // create a note and a paragraph Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p = note.addParagraph(); Map config = new HashMap<String, Object>(); @@ -295,11 +295,11 @@ public class NotebookTest implements JobListenerFactory{ MockInterpreter1 mock1 = ((MockInterpreter1) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) note.getNoteReplLoader().get("mock1")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock1")).getInnerInterpreter()) .getInnerInterpreter())); MockInterpreter2 mock2 = ((MockInterpreter2) (((ClassloaderInterpreter) - ((LazyOpenInterpreter) note.getNoteReplLoader().get("mock2")).getInnerInterpreter()) + ((LazyOpenInterpreter) factory.getInterpreter(note.getId(), "mock2")).getInnerInterpreter()) .getInnerInterpreter())); // wait until interpreters are started @@ -326,7 +326,7 @@ public class NotebookTest implements JobListenerFactory{ public void testExportAndImportNote() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); String simpleText = "hello world"; @@ -353,7 +353,7 @@ public class NotebookTest implements JobListenerFactory{ public void testCloneNote() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); p.setText("hello world"); @@ -375,7 +375,7 @@ public class NotebookTest implements JobListenerFactory{ public void testCloneNoteWithExceptionResult() throws IOException, CloneNotSupportedException, InterruptedException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); final Paragraph p = note.addParagraph(); p.setText("hello world"); @@ -398,7 +398,7 @@ public class NotebookTest implements JobListenerFactory{ @Test public void testResourceRemovealOnParagraphNoteRemove() throws IOException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) { intpGroup.setResourcePool(new LocalResourcePool(intpGroup.getId())); } @@ -427,10 +427,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = note.getNoteReplLoader() - .getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess") + AngularObjectRegistry registry = factory + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addParagraph(); @@ -460,10 +460,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = note.getNoteReplLoader() - .getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess") + AngularObjectRegistry registry = factory + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") .getAngularObjectRegistry(); Paragraph p1 = note.addParagraph(); @@ -493,10 +493,10 @@ public class NotebookTest implements JobListenerFactory{ IOException { // create a note and a paragraph Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); - AngularObjectRegistry registry = note.getNoteReplLoader() - .getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess") + AngularObjectRegistry registry = factory + .getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") .getAngularObjectRegistry(); // add local scope object @@ -505,9 +505,8 @@ public class NotebookTest implements JobListenerFactory{ registry.add("o2", "object2", null, null); // restart interpreter - factory.restart(note.getNoteReplLoader().getInterpreterSettings().get(0).id()); - registry = note.getNoteReplLoader() - .getInterpreterSettings().get(0).getInterpreterGroup("sharedProcess") + factory.restart(factory.getInterpreterSettings(note.getId()).get(0).id()); + registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") .getAngularObjectRegistry(); // local and global scope object should be removed @@ -565,7 +564,7 @@ public class NotebookTest implements JobListenerFactory{ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedException, IOException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); ArrayList<Paragraph> paragraphs = new ArrayList<>(); for (int i = 0; i < 100; i++) { @@ -582,7 +581,7 @@ public class NotebookTest implements JobListenerFactory{ while (paragraphs.get(0).getStatus() != Status.FINISHED) Thread.yield(); - factory.restart(note.getNoteReplLoader().getInterpreterSettings().get(0).id()); + factory.restart(factory.getInterpreterSettings(note.getId()).get(0).id()); boolean isAborted = false; for (Paragraph p : paragraphs) { @@ -606,7 +605,7 @@ public class NotebookTest implements JobListenerFactory{ p1.setText("getId"); // restart interpreter with per note session enabled - for (InterpreterSetting setting : note1.getNoteReplLoader().getInterpreterSettings()) { + for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); notebook.getInterpreterFactory().restart(setting.id()); } @@ -651,7 +650,7 @@ public class NotebookTest implements JobListenerFactory{ // restart interpreter with per note session enabled - for (InterpreterSetting setting : note1.getNoteReplLoader().getInterpreterSettings()) { + for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); notebook.getInterpreterFactory().restart(setting.id()); } @@ -677,7 +676,7 @@ public class NotebookTest implements JobListenerFactory{ p1.setText("getId"); // restart interpreter with per note session enabled - for (InterpreterSetting setting : note1.getNoteReplLoader().getInterpreterSettings()) { + for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); notebook.getInterpreterFactory().restart(setting.id()); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java index 833eef3..1f8519c 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/ParagraphTest.java @@ -19,6 +19,8 @@ package org.apache.zeppelin.notebook; import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -28,6 +30,7 @@ import org.apache.zeppelin.display.AngularObjectBuilder; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterFactory; import org.junit.Test; import java.util.HashMap; @@ -72,17 +75,19 @@ public class ParagraphTest { @Test public void effectiveTextTest() { - NoteInterpreterLoader noteInterpreterLoader = mock(NoteInterpreterLoader.class); + InterpreterFactory interpreterFactory = mock(InterpreterFactory.class); Interpreter interpreter = mock(Interpreter.class); + Note note = mock(Note.class); - Paragraph p = new Paragraph(null, null, null, noteInterpreterLoader); + Paragraph p = new Paragraph("paragraph", note, null, interpreterFactory); p.setText("%h2 show databases"); p.setEffectiveText("%jdbc(h2) show databases"); assertEquals("Get right replName", "jdbc", p.getRequiredReplName()); assertEquals("Get right scriptBody", "(h2) show databases", p.getScriptBody()); - when(noteInterpreterLoader.get("jdbc")).thenReturn(interpreter); + when(interpreterFactory.getInterpreter(anyString(), eq("jdbc"))).thenReturn(interpreter); when(interpreter.getFormType()).thenReturn(Interpreter.FormType.NATIVE); + when(note.getId()).thenReturn("noteId"); try { p.jobRun(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java index 0550e90..9fb0360 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepoTest.java @@ -107,7 +107,7 @@ public class VFSNotebookRepoTest implements JobListenerFactory { @Test public void testSaveNotebook() throws IOException, InterruptedException { Note note = notebook.createNote(null); - note.getNoteReplLoader().setInterpreters(factory.getDefaultInterpreterSettingList()); + factory.setInterpreters(note.getId(), factory.getDefaultInterpreterSettingList()); Paragraph p1 = note.addParagraph(); Map<String, Object> config = p1.getConfig(); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/2a2a2e82/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java index 3641929..b7fa2df 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/search/LuceneSearchTest.java @@ -25,9 +25,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.InterpreterFactory; import org.apache.zeppelin.notebook.Note; -import org.apache.zeppelin.notebook.NoteInterpreterLoader; import org.apache.zeppelin.notebook.Paragraph; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.junit.After; @@ -36,21 +35,20 @@ import org.junit.BeforeClass; import org.junit.Test; import com.google.common.base.Splitter; -import com.google.common.collect.ImmutableList; public class LuceneSearchTest { - private static NoteInterpreterLoader replLoaderMock; private static NotebookRepo notebookRepoMock; + private static InterpreterFactory interpreterFactory; private SearchService notebookIndex; @BeforeClass public static void beforeStartUp() { notebookRepoMock = mock(NotebookRepo.class); - replLoaderMock = mock(NoteInterpreterLoader.class); + interpreterFactory = mock(InterpreterFactory.class); - when(replLoaderMock.getInterpreterSettings()) - .thenReturn(ImmutableList.<InterpreterSetting>of()); +// when(replLoaderMock.getInterpreterSettings()) +// .thenReturn(ImmutableList.<InterpreterSetting>of()); } @Before @@ -286,7 +284,7 @@ public class LuceneSearchTest { } private Note newNote(String name) { - Note note = new Note(notebookRepoMock, replLoaderMock, null, notebookIndex, null); + Note note = new Note(notebookRepoMock, interpreterFactory, null, notebookIndex, null); note.setName(name); return note; }
