http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java new file mode 100644 index 0000000..b141e6b --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -0,0 +1,1125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.internal.StringMap; +import com.google.gson.reflect.TypeToken; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; +import org.apache.zeppelin.dep.Dependency; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.scheduler.Job; +import org.apache.zeppelin.scheduler.Job.Status; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; +import org.sonatype.aether.repository.RemoteRepository; + +/** + * TBD + */ +public class InterpreterSettingManager { + + private static final Logger logger = LoggerFactory.getLogger(InterpreterSettingManager.class); + private static final String SHARED_SESSION = "shared_session"; + private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of( + "language", (Object) "text", + "editOnDblClick", false); + + private final ZeppelinConfiguration zeppelinConfiguration; + private final Path interpreterDirPath; + private final Path interpreterBindingPath; + + /** + * This is only references with default settings, name and properties + * key: InterpreterSetting.name + */ + private final Map<String, InterpreterSetting> interpreterSettingsRef; + /** + * This is used by creating and running Interpreters + * key: InterpreterSetting.id <- This is becuase backward compatibility + */ + private final Map<String, InterpreterSetting> interpreterSettings; + private final Map<String, List<String>> interpreterBindings; + + private final DependencyResolver dependencyResolver; + private final List<RemoteRepository> interpreterRepositories; + + private final InterpreterOption defaultOption; + + private final Map<String, URLClassLoader> cleanCl; + + @Deprecated + private String[] interpreterClassList; + private String[] interpreterGroupOrderList; + private InterpreterGroupFactory interpreterGroupFactory; + + private final Gson gson; + + public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, + DependencyResolver dependencyResolver, InterpreterOption interpreterOption) + throws IOException, RepositoryException { + this.zeppelinConfiguration = zeppelinConfiguration; + this.interpreterDirPath = Paths.get(zeppelinConfiguration.getInterpreterDir()); + logger.debug("InterpreterRootPath: {}", interpreterDirPath); + this.interpreterBindingPath = Paths.get(zeppelinConfiguration.getInterpreterSettingPath()); + logger.debug("InterpreterBindingPath: {}", interpreterBindingPath); + + this.interpreterSettingsRef = Maps.newConcurrentMap(); + this.interpreterSettings = Maps.newConcurrentMap(); + this.interpreterBindings = Maps.newConcurrentMap(); + + this.dependencyResolver = dependencyResolver; + this.interpreterRepositories = dependencyResolver.getRepos(); + + this.defaultOption = interpreterOption; + + this.cleanCl = Collections.synchronizedMap(new HashMap<String, URLClassLoader>()); + + String replsConf = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETERS); + this.interpreterClassList = replsConf.split(","); + String groupOrder = zeppelinConfiguration.getString(ConfVars.ZEPPELIN_INTERPRETER_GROUP_ORDER); + this.interpreterGroupOrderList = groupOrder.split(","); + + GsonBuilder gsonBuilder = new GsonBuilder(); + gsonBuilder.setPrettyPrinting(); + this.gson = gsonBuilder.create(); + + init(); + } + + private void loadFromFile() { + if (!Files.exists(interpreterBindingPath)) { + // nothing to read + return; + } + InterpreterInfoSaving infoSaving; + try (BufferedReader json = + Files.newBufferedReader(interpreterBindingPath, StandardCharsets.UTF_8)) { + infoSaving = gson.fromJson(json, InterpreterInfoSaving.class); + + for (String k : infoSaving.interpreterSettings.keySet()) { + InterpreterSetting setting = infoSaving.interpreterSettings.get(k); + List<InterpreterInfo> infos = setting.getInterpreterInfos(); + + // Convert json StringMap to Properties + StringMap<String> p = (StringMap<String>) setting.getProperties(); + Properties properties = new Properties(); + for (String key : p.keySet()) { + properties.put(key, p.get(key)); + } + setting.setProperties(properties); + + // Always use separate interpreter process + // While we decided to turn this feature on always (without providing + // enable/disable option on GUI). + // previously created setting should turn this feature on here. + setting.getOption().setRemote(true); + + // Update transient information from InterpreterSettingRef + InterpreterSetting interpreterSettingObject = + interpreterSettingsRef.get(setting.getGroup()); + if (interpreterSettingObject == null) { + logger.warn("can't get InterpreterSetting " + + "Information From loaded Interpreter Setting Ref - {} ", setting.getGroup()); + continue; + } + String depClassPath = interpreterSettingObject.getPath(); + setting.setPath(depClassPath); + + for (InterpreterInfo info : infos) { + if (info.getEditor() == null) { + Map<String, Object> editor = getEditorFromSettingByClassName(interpreterSettingObject, + info.getClassName()); + info.setEditor(editor); + } + } + + setting.setInterpreterGroupFactory(interpreterGroupFactory); + + loadInterpreterDependencies(setting); + interpreterSettings.put(k, setting); + } + + interpreterBindings.putAll(infoSaving.interpreterBindings); + + if (infoSaving.interpreterRepositories != null) { + for (RemoteRepository repo : infoSaving.interpreterRepositories) { + if (!dependencyResolver.getRepos().contains(repo)) { + this.interpreterRepositories.add(repo); + } + } + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void saveToFile() throws IOException { + String jsonString; + + synchronized (interpreterSettings) { + InterpreterInfoSaving info = new InterpreterInfoSaving(); + info.interpreterBindings = interpreterBindings; + info.interpreterSettings = interpreterSettings; + info.interpreterRepositories = interpreterRepositories; + + jsonString = gson.toJson(info); + } + + if (!Files.exists(interpreterBindingPath)) { + Files.createFile(interpreterBindingPath); + + Set<PosixFilePermission> permissions = EnumSet.of(OWNER_READ, OWNER_WRITE); + Files.setPosixFilePermissions(interpreterBindingPath, permissions); + } + + FileOutputStream fos = new FileOutputStream(interpreterBindingPath.toFile(), false); + OutputStreamWriter out = new OutputStreamWriter(fos); + out.append(jsonString); + out.close(); + fos.close(); + } + + //TODO(jl): Fix it to remove InterpreterGroupFactory + public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) { + for (InterpreterSetting setting : interpreterSettings.values()) { + setting.setInterpreterGroupFactory(interpreterGroupFactory); + } + this.interpreterGroupFactory = interpreterGroupFactory; + } + + private void init() throws InterpreterException, IOException, RepositoryException { + String interpreterJson = zeppelinConfiguration.getInterpreterJson(); + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + + if (Files.exists(interpreterDirPath)) { + for (Path interpreterDir : Files + .newDirectoryStream(interpreterDirPath, new Filter<Path>() { + @Override + public boolean accept(Path entry) throws IOException { + return Files.exists(entry) && Files.isDirectory(entry); + } + })) { + String interpreterDirString = interpreterDir.toString(); + + /** + * Register interpreter by the following ordering + * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ + * interpreter-setting.json + * 2. Register it from interpreter-setting.json in classpath + * {ZEPPELIN_HOME}/interpreter/{interpreter_name} + * 3. Register it by Interpreter.register + */ + if (!registerInterpreterFromPath(interpreterDirString, interpreterJson)) { + if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson)) { + /* + * TODO(jongyoul) + * - Remove these codes below because of legacy code + * - Support ThreadInterpreter + */ + URLClassLoader ccl = new URLClassLoader( + recursiveBuildLibList(interpreterDir.toFile()), cl); + for (String className : interpreterClassList) { + try { + // Load classes + Class.forName(className, true, ccl); + Set<String> interpreterKeys = Interpreter.registeredInterpreters.keySet(); + for (String interpreterKey : interpreterKeys) { + if (className + .equals(Interpreter.registeredInterpreters.get(interpreterKey) + .getClassName())) { + Interpreter.registeredInterpreters.get(interpreterKey) + .setPath(interpreterDirString); + logger.info("Interpreter " + interpreterKey + " found. class=" + className); + cleanCl.put(interpreterDirString, ccl); + } + } + } catch (Throwable t) { + // nothing to do + } + } + } + } + } + } + + for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters + .values()) { + logger + .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(), + registeredInterpreter.getClassName(), registeredInterpreter.getProperties()); + } + + // RegisteredInterpreters -> interpreterSettingRef + InterpreterInfo interpreterInfo; + for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) { + interpreterInfo = + new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter(), + r.getEditor()); + add(r.getGroup(), interpreterInfo, r.getProperties(), defaultOption, r.getPath(), + r.getRunner()); + } + + for (String settingId : interpreterSettingsRef.keySet()) { + InterpreterSetting setting = interpreterSettingsRef.get(settingId); + logger.info("InterpreterSettingRef name {}", setting.getName()); + } + + loadFromFile(); + + // if no interpreter settings are loaded, create default set + if (0 == interpreterSettings.size()) { + Map<String, InterpreterSetting> temp = new HashMap<>(); + InterpreterSetting interpreterSetting; + for (InterpreterSetting setting : interpreterSettingsRef.values()) { + interpreterSetting = createFromInterpreterSettingRef(setting); + temp.put(setting.getName(), interpreterSetting); + } + + for (String group : interpreterGroupOrderList) { + if (null != (interpreterSetting = temp.remove(group))) { + interpreterSettings.put(interpreterSetting.getId(), interpreterSetting); + } + } + + for (InterpreterSetting setting : temp.values()) { + interpreterSettings.put(setting.getId(), setting); + } + + saveToFile(); + } + + for (String settingId : interpreterSettings.keySet()) { + InterpreterSetting setting = interpreterSettings.get(settingId); + logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId, + setting.getName()); + } + } + + private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, + String interpreterJson) throws IOException, RepositoryException { + URL[] urls = recursiveBuildLibList(new File(interpreterDir)); + ClassLoader tempClassLoader = new URLClassLoader(urls, cl); + + Enumeration<URL> interpreterSettings = tempClassLoader.getResources(interpreterJson); + if (!interpreterSettings.hasMoreElements()) { + return false; + } + for (URL url : Collections.list(interpreterSettings)) { + try (InputStream inputStream = url.openStream()) { + logger.debug("Reading {} from {}", interpreterJson, url); + List<RegisteredInterpreter> registeredInterpreterList = + getInterpreterListFromJson(inputStream); + registerInterpreters(registeredInterpreterList, interpreterDir); + } + } + return true; + } + + private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson) + throws IOException, RepositoryException { + + Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); + if (Files.exists(interpreterJsonPath)) { + logger.debug("Reading {}", interpreterJsonPath); + List<RegisteredInterpreter> registeredInterpreterList = + getInterpreterListFromJson(interpreterJsonPath); + registerInterpreters(registeredInterpreterList, interpreterDir); + return true; + } + return false; + } + + private List<RegisteredInterpreter> getInterpreterListFromJson(Path filename) + throws FileNotFoundException { + return getInterpreterListFromJson(new FileInputStream(filename.toFile())); + } + + private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) { + Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() { + }.getType(); + return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType); + } + + private void registerInterpreters(List<RegisteredInterpreter> registeredInterpreters, + String absolutePath) throws IOException, RepositoryException { + + for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { + InterpreterInfo interpreterInfo = + new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), + registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor()); + // use defaultOption if it is not specified in interpreter-setting.json + InterpreterOption option = registeredInterpreter.getOption() == null ? defaultOption : + registeredInterpreter.getOption(); + add(registeredInterpreter.getGroup(), interpreterInfo, registeredInterpreter.getProperties(), + option, absolutePath, registeredInterpreter.getRunner()); + } + + } + + public InterpreterSetting getDefaultInterpreterSetting(List<InterpreterSetting> settings) { + if (settings == null || settings.isEmpty()) { + return null; + } + return settings.get(0); + } + + public InterpreterSetting getDefaultInterpreterSetting(String noteId) { + return getDefaultInterpreterSetting(getInterpreterSettings(noteId)); + } + + public List<InterpreterSetting> getInterpreterSettings(String noteId) { + List<String> interpreterSettingIds = getNoteInterpreterSettingBinding(noteId); + LinkedList<InterpreterSetting> settings = new LinkedList<>(); + + Iterator<String> iter = interpreterSettingIds.iterator(); + while (iter.hasNext()) { + String id = iter.next(); + InterpreterSetting setting = get(id); + if (setting == null) { + // interpreter setting is removed from factory. remove id from here, too + iter.remove(); + } else { + settings.add(setting); + } + } + return settings; + } + + private List<String> getNoteInterpreterSettingBinding(String noteId) { + LinkedList<String> bindings = new LinkedList<>(); + synchronized (interpreterSettings) { + List<String> settingIds = interpreterBindings.get(noteId); + if (settingIds != null) { + bindings.addAll(settingIds); + } + } + return bindings; + } + + private InterpreterSetting createFromInterpreterSettingRef(String name) { + Preconditions.checkNotNull(name, "reference name should be not null"); + InterpreterSetting settingRef = interpreterSettingsRef.get(name); + return createFromInterpreterSettingRef(settingRef); + } + + private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) { + // should return immutable objects + List<InterpreterInfo> infos = (null == o.getInterpreterInfos()) ? + new ArrayList<InterpreterInfo>() : new ArrayList<>(o.getInterpreterInfos()); + List<Dependency> deps = (null == o.getDependencies()) ? + new ArrayList<Dependency>() : new ArrayList<>(o.getDependencies()); + Properties props = + convertInterpreterProperties((Map<String, InterpreterProperty>) o.getProperties()); + InterpreterOption option = InterpreterOption.fromInterpreterOption(o.getOption()); + + InterpreterSetting setting = new InterpreterSetting(o.getName(), o.getName(), + infos, props, deps, option, o.getPath(), o.getInterpreterRunner()); + setting.setInterpreterGroupFactory(interpreterGroupFactory); + return setting; + } + + private Properties convertInterpreterProperties(Map<String, InterpreterProperty> p) { + Properties properties = new Properties(); + for (String key : p.keySet()) { + properties.put(key, p.get(key).getValue()); + } + return properties; + } + + public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId, + String replName) { + Map<String, Object> editor = DEFAULT_EDITOR; + String group = StringUtils.EMPTY; + try { + String defaultSettingName = getDefaultInterpreterSetting(noteId).getName(); + List<InterpreterSetting> intpSettings = getInterpreterSettings(noteId); + for (InterpreterSetting intpSetting : intpSettings) { + String[] replNameSplit = replName.split("\\."); + if (replNameSplit.length == 2) { + group = replNameSplit[0]; + } + // when replName is 'name' of interpreter + if (defaultSettingName.equals(intpSetting.getName())) { + editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName()); + } + // when replName is 'alias name' of interpreter or 'group' of interpreter + if (replName.equals(intpSetting.getName()) || group.equals(intpSetting.getName())) { + editor = getEditorFromSettingByClassName(intpSetting, interpreter.getClassName()); + break; + } + } + } catch (NullPointerException e) { + logger.warn("Couldn't get interpreter editor setting"); + } + return editor; + } + + public Map<String, Object> getEditorFromSettingByClassName(InterpreterSetting intpSetting, + String className) { + List<InterpreterInfo> intpInfos = intpSetting.getInterpreterInfos(); + for (InterpreterInfo intpInfo : intpInfos) { + + if (className.equals(intpInfo.getClassName())) { + if (intpInfo.getEditor() == null) { + break; + } + return intpInfo.getEditor(); + } + } + return DEFAULT_EDITOR; + } + + private void loadInterpreterDependencies(final InterpreterSetting setting) { + setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); + setting.setErrorReason(null); + interpreterSettings.put(setting.getId(), setting); + synchronized (interpreterSettings) { + final Thread t = new Thread() { + public void run() { + try { + // dependencies to prevent library conflict + File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + + setting.getId()); + if (localRepoDir.exists()) { + try { + FileUtils.cleanDirectory(localRepoDir); + } catch (FileNotFoundException e) { + logger.info("A file that does not exist cannot be deleted, nothing to worry", e); + } + } + + // load dependencies + List<Dependency> deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + if (d.getExclusions() != null) { + dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), + new File(destDir, setting.getId())); + } else { + dependencyResolver + .load(d.getGroupArtifactVersion(), new File(destDir, setting.getId())); + } + } + } + + setting.setStatus(InterpreterSetting.Status.READY); + setting.setErrorReason(null); + } catch (Exception e) { + logger.error(String.format("Error while downloading repos for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly. : %s", + setting.getGroup(), e.getLocalizedMessage()), e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + interpreterSettings.put(setting.getId(), setting); + } + } + }; + t.start(); + } + } + + /** + * Overwrite dependency jar under local-repo/{interpreterId} + * if jar file in original path is changed + */ + private void copyDependenciesFromLocalPath(final InterpreterSetting setting) { + setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); + interpreterSettings.put(setting.getId(), setting); + synchronized (interpreterSettings) { + final Thread t = new Thread() { + public void run() { + try { + List<Dependency> deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File( + zeppelinConfiguration.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + int numSplits = d.getGroupArtifactVersion().split(":").length; + if (!(numSplits >= 3 && numSplits <= 6)) { + dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(), + new File(destDir, setting.getId())); + } + } + } + setting.setStatus(InterpreterSetting.Status.READY); + } catch (Exception e) { + logger.error(String.format("Error while copying deps for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly.", + setting.getGroup()), e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + interpreterSettings.put(setting.getId(), setting); + } + } + }; + t.start(); + } + } + + /** + * Return ordered interpreter setting list. + * The list does not contain more than one setting from the same interpreter class. + * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name + */ + public List<String> getDefaultInterpreterSettingList() { + // this list will contain default interpreter setting list + List<String> defaultSettings = new LinkedList<>(); + + // to ignore the same interpreter group + Map<String, Boolean> interpreterGroupCheck = new HashMap<>(); + + List<InterpreterSetting> sortedSettings = get(); + + for (InterpreterSetting setting : sortedSettings) { + if (defaultSettings.contains(setting.getId())) { + continue; + } + + if (!interpreterGroupCheck.containsKey(setting.getName())) { + defaultSettings.add(setting.getId()); + interpreterGroupCheck.put(setting.getName(), true); + } + } + return defaultSettings; + } + + List<RegisteredInterpreter> getRegisteredInterpreterList() { + return new ArrayList<>(Interpreter.registeredInterpreters.values()); + } + + + private boolean findDefaultInterpreter(List<InterpreterInfo> infos) { + for (InterpreterInfo interpreterInfo : infos) { + if (interpreterInfo.isDefaultInterpreter()) { + return true; + } + } + return false; + } + + public InterpreterSetting createNewSetting(String name, String group, + List<Dependency> dependencies, InterpreterOption option, Properties p) throws IOException { + if (name.indexOf(".") >= 0) { + throw new IOException("'.' is invalid for InterpreterSetting name."); + } + InterpreterSetting setting = createFromInterpreterSettingRef(group); + setting.setName(name); + setting.setGroup(group); + setting.appendDependencies(dependencies); + setting.setInterpreterOption(option); + setting.setProperties(p); + setting.setInterpreterGroupFactory(interpreterGroupFactory); + interpreterSettings.put(setting.getId(), setting); + loadInterpreterDependencies(setting); + saveToFile(); + return setting; + } + + private InterpreterSetting add(String group, InterpreterInfo interpreterInfo, + Map<String, InterpreterProperty> interpreterProperties, InterpreterOption option, String path, + InterpreterRunner runner) + throws InterpreterException, IOException, RepositoryException { + ArrayList<InterpreterInfo> infos = new ArrayList<>(); + infos.add(interpreterInfo); + return add(group, infos, new ArrayList<Dependency>(), option, interpreterProperties, path, + runner); + } + + /** + * @param group InterpreterSetting reference name + */ + public InterpreterSetting add(String group, ArrayList<InterpreterInfo> interpreterInfos, + List<Dependency> dependencies, InterpreterOption option, + Map<String, InterpreterProperty> interpreterProperties, String path, + InterpreterRunner runner) { + Preconditions.checkNotNull(group, "name should not be null"); + Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null"); + Preconditions.checkNotNull(dependencies, "dependencies should not be null"); + Preconditions.checkNotNull(option, "option should not be null"); + Preconditions.checkNotNull(interpreterProperties, "properties should not be null"); + + InterpreterSetting interpreterSetting; + + synchronized (interpreterSettingsRef) { + if (interpreterSettingsRef.containsKey(group)) { + interpreterSetting = interpreterSettingsRef.get(group); + + // Append InterpreterInfo + List<InterpreterInfo> infos = interpreterSetting.getInterpreterInfos(); + boolean hasDefaultInterpreter = findDefaultInterpreter(infos); + for (InterpreterInfo interpreterInfo : interpreterInfos) { + if (!infos.contains(interpreterInfo)) { + if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) { + hasDefaultInterpreter = true; + infos.add(0, interpreterInfo); + } else { + infos.add(interpreterInfo); + } + } + } + + // Append dependencies + List<Dependency> dependencyList = interpreterSetting.getDependencies(); + for (Dependency dependency : dependencies) { + if (!dependencyList.contains(dependency)) { + dependencyList.add(dependency); + } + } + + // Append properties + Map<String, InterpreterProperty> properties = + (Map<String, InterpreterProperty>) interpreterSetting.getProperties(); + for (String key : interpreterProperties.keySet()) { + if (!properties.containsKey(key)) { + properties.put(key, interpreterProperties.get(key)); + } + } + + } else { + interpreterSetting = + new InterpreterSetting(group, null, interpreterInfos, interpreterProperties, + dependencies, option, path, runner); + interpreterSettingsRef.put(group, interpreterSetting); + } + } + + if (dependencies.size() > 0) { + loadInterpreterDependencies(interpreterSetting); + } + + interpreterSetting.setInterpreterGroupFactory(interpreterGroupFactory); + return interpreterSetting; + } + + /** + * map interpreter ids into noteId + * + * @param noteId note id + * @param ids InterpreterSetting id list + */ + public void setInterpreters(String user, String noteId, List<String> ids) throws IOException { + putNoteInterpreterSettingBinding(user, noteId, ids); + } + + private void putNoteInterpreterSettingBinding(String user, String noteId, + List<String> settingList) throws IOException { + List<String> unBindedSettings = new LinkedList<>(); + + synchronized (interpreterSettings) { + List<String> oldSettings = interpreterBindings.get(noteId); + if (oldSettings != null) { + for (String oldSettingId : oldSettings) { + if (!settingList.contains(oldSettingId)) { + unBindedSettings.add(oldSettingId); + } + } + } + interpreterBindings.put(noteId, settingList); + saveToFile(); + + for (String settingId : unBindedSettings) { + InterpreterSetting setting = get(settingId); + removeInterpretersForNote(setting, user, noteId); + } + } + } + + public void removeInterpretersForNote(InterpreterSetting interpreterSetting, String user, + String noteId) { + InterpreterOption option = interpreterSetting.getOption(); + if (option.isProcess()) { + interpreterSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); + } else if (option.isSession()) { + InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(user, noteId); + String key = getInterpreterSessionKey(user, noteId, interpreterSetting); + interpreterGroup.close(key); + synchronized (interpreterGroup) { + interpreterGroup.remove(key); + interpreterGroup.notifyAll(); // notify createInterpreterForNote() + } + logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), + noteId); + } + } + + public String getInterpreterSessionKey(String user, String noteId, InterpreterSetting setting) { + InterpreterOption option = setting.getOption(); + String key; + if (option.isExistingProcess()) { + key = Constants.EXISTING_PROCESS; + } else if (option.perNoteScoped() && option.perUserScoped()) { + key = user + ":" + noteId; + } else if (option.perUserScoped()) { + key = user; + } else if (option.perNoteScoped()) { + key = noteId; + } else { + key = SHARED_SESSION; + } + + logger.debug("Interpreter session key: {}, for note: {}, user: {}, InterpreterSetting Name: " + + "{}", key, noteId, user, setting.getName()); + return key; + } + + + public List<String> getInterpreters(String noteId) { + return getNoteInterpreterSettingBinding(noteId); + } + + public void closeNote(String user, String noteId) { + // close interpreters in this note session + List<InterpreterSetting> settings = getInterpreterSettings(noteId); + if (settings == null || settings.size() == 0) { + return; + } + + logger.info("closeNote: {}", noteId); + for (InterpreterSetting setting : settings) { + removeInterpretersForNote(setting, user, noteId); + } + } + + public Map<String, InterpreterSetting> getAvailableInterpreterSettings() { + return interpreterSettingsRef; + } + + private URL[] recursiveBuildLibList(File path) throws MalformedURLException { + URL[] urls = new URL[0]; + if (path == null || !path.exists()) { + return urls; + } else if (path.getName().startsWith(".")) { + return urls; + } else if (path.isDirectory()) { + File[] files = path.listFiles(); + if (files != null) { + for (File f : files) { + urls = (URL[]) ArrayUtils.addAll(urls, recursiveBuildLibList(f)); + } + } + return urls; + } else { + return new URL[]{path.toURI().toURL()}; + } + } + + public List<RemoteRepository> getRepositories() { + return this.interpreterRepositories; + } + + public void addRepository(String id, String url, boolean snapshot, Authentication auth, + Proxy proxy) throws IOException { + dependencyResolver.addRepo(id, url, snapshot, auth, proxy); + saveToFile(); + } + + public void removeRepository(String id) throws IOException { + dependencyResolver.delRepo(id); + saveToFile(); + } + + public void removeNoteInterpreterSettingBinding(String user, String noteId) { + synchronized (interpreterSettings) { + List<String> settingIds = (interpreterBindings.containsKey(noteId) ? + interpreterBindings.remove(noteId) : + Collections.<String>emptyList()); + for (String settingId : settingIds) { + this.removeInterpretersForNote(get(settingId), user, noteId); + } + } + } + + /** + * Change interpreter property and restart + */ + public void setPropertyAndRestart(String id, InterpreterOption option, Properties properties, + List<Dependency> dependencies) throws IOException { + synchronized (interpreterSettings) { + InterpreterSetting intpSetting = interpreterSettings.get(id); + if (intpSetting != null) { + try { + stopJobAllInterpreter(intpSetting); + + intpSetting.closeAndRemoveAllInterpreterGroups(); + intpSetting.setOption(option); + intpSetting.setProperties(properties); + intpSetting.setDependencies(dependencies); + loadInterpreterDependencies(intpSetting); + + saveToFile(); + } catch (Exception e) { + throw e; + } finally { + loadFromFile(); + } + } else { + throw new InterpreterException("Interpreter setting id " + id + " not found"); + } + } + } + + public void restart(String settingId, String noteId, String user) { + InterpreterSetting intpSetting = interpreterSettings.get(settingId); + Preconditions.checkNotNull(intpSetting); + + // restart interpreter setting in note page + if (noteIdIsExist(noteId) && intpSetting.getOption().isProcess()) { + intpSetting.closeAndRemoveInterpreterGroupByNoteId(noteId); + return; + } else { + // restart interpreter setting in interpreter setting page + restart(settingId, user); + } + + } + + private boolean noteIdIsExist(String noteId) { + return noteId == null ? false : true; + } + + public void restart(String id, String user) { + synchronized (interpreterSettings) { + InterpreterSetting intpSetting = interpreterSettings.get(id); + // Check if dependency in specified path is changed + // If it did, overwrite old dependency jar with new one + if (intpSetting != null) { + //clean up metaInfos + intpSetting.setInfos(null); + copyDependenciesFromLocalPath(intpSetting); + + stopJobAllInterpreter(intpSetting); + if (user.equals("anonymous")) { + intpSetting.closeAndRemoveAllInterpreterGroups(); + } else { + intpSetting.closeAndRemoveInterpreterGroupByUser(user); + } + + } else { + throw new InterpreterException("Interpreter setting id " + id + " not found"); + } + } + } + + public void restart(String id) { + restart(id, "anonymous"); + } + + private void stopJobAllInterpreter(InterpreterSetting intpSetting) { + if (intpSetting != null) { + for (InterpreterGroup intpGroup : intpSetting.getAllInterpreterGroups()) { + for (List<Interpreter> interpreters : intpGroup.values()) { + for (Interpreter intp : interpreters) { + for (Job job : intp.getScheduler().getJobsRunning()) { + job.abort(); + job.setStatus(Status.ABORT); + logger.info("Job " + job.getJobName() + " aborted "); + } + for (Job job : intp.getScheduler().getJobsWaiting()) { + job.abort(); + job.setStatus(Status.ABORT); + logger.info("Job " + job.getJobName() + " aborted "); + } + } + } + } + } + } + + public InterpreterSetting get(String name) { + synchronized (interpreterSettings) { + return interpreterSettings.get(name); + } + } + + public void remove(String id) throws IOException { + synchronized (interpreterSettings) { + if (interpreterSettings.containsKey(id)) { + InterpreterSetting intp = interpreterSettings.get(id); + intp.closeAndRemoveAllInterpreterGroups(); + + interpreterSettings.remove(id); + for (List<String> settings : interpreterBindings.values()) { + Iterator<String> it = settings.iterator(); + while (it.hasNext()) { + String settingId = it.next(); + if (settingId.equals(id)) { + it.remove(); + } + } + } + saveToFile(); + } + } + + File localRepoDir = new File(zeppelinConfiguration.getInterpreterLocalRepoPath() + "/" + id); + FileUtils.deleteDirectory(localRepoDir); + } + + /** + * Get interpreter settings + */ + public List<InterpreterSetting> get() { + synchronized (interpreterSettings) { + List<InterpreterSetting> orderedSettings = new LinkedList<>(); + + Map<String, List<InterpreterSetting>> nameInterpreterSettingMap = new HashMap<>(); + for (InterpreterSetting interpreterSetting : interpreterSettings.values()) { + String group = interpreterSetting.getGroup(); + if (!nameInterpreterSettingMap.containsKey(group)) { + nameInterpreterSettingMap.put(group, new ArrayList<InterpreterSetting>()); + } + nameInterpreterSettingMap.get(group).add(interpreterSetting); + } + + for (String groupName : interpreterGroupOrderList) { + List<InterpreterSetting> interpreterSettingList = + nameInterpreterSettingMap.remove(groupName); + if (null != interpreterSettingList) { + for (InterpreterSetting interpreterSetting : interpreterSettingList) { + orderedSettings.add(interpreterSetting); + } + } + } + + List<InterpreterSetting> settings = new ArrayList<>(); + + for (List<InterpreterSetting> interpreterSettingList : nameInterpreterSettingMap.values()) { + for (InterpreterSetting interpreterSetting : interpreterSettingList) { + settings.add(interpreterSetting); + } + } + + Collections.sort(settings, new Comparator<InterpreterSetting>() { + @Override + public int compare(InterpreterSetting o1, InterpreterSetting o2) { + return o1.getName().compareTo(o2.getName()); + } + }); + + orderedSettings.addAll(settings); + + return orderedSettings; + } + } + + public void close() { + List<Thread> closeThreads = new LinkedList<>(); + synchronized (interpreterSettings) { + Collection<InterpreterSetting> intpSettings = interpreterSettings.values(); + for (final InterpreterSetting intpSetting : intpSettings) { + Thread t = new Thread() { + public void run() { + intpSetting.closeAndRemoveAllInterpreterGroups(); + } + }; + t.start(); + closeThreads.add(t); + } + } + + for (Thread t : closeThreads) { + try { + t.join(); + } catch (InterruptedException e) { + logger.error("Can't close interpreterGroup", e); + } + } + } + + public void shutdown() { + List<Thread> closeThreads = new LinkedList<>(); + synchronized (interpreterSettings) { + Collection<InterpreterSetting> intpSettings = interpreterSettings.values(); + for (final InterpreterSetting intpSetting : intpSettings) { + Thread t = new Thread() { + public void run() { + intpSetting.shutdownAndRemoveAllInterpreterGroups(); + } + }; + t.start(); + closeThreads.add(t); + } + } + + for (Thread t : closeThreads) { + try { + t.join(); + } catch (InterruptedException e) { + logger.error("Can't close interpreterGroup", e); + } + } + } +}
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 73279ff..35f32f3 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -77,6 +77,7 @@ public class Note implements Serializable, ParagraphJobListener { private Map<String, List<AngularObject>> angularObjects = new HashMap<>(); private transient InterpreterFactory factory; + private transient InterpreterSettingManager interpreterSettingManager; private transient JobListenerFactory jobListenerFactory; private transient NotebookRepo repo; private transient SearchService index; @@ -101,10 +102,12 @@ public class Note implements Serializable, ParagraphJobListener { public Note() { } - public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory, + public Note(NotebookRepo repo, InterpreterFactory factory, + InterpreterSettingManager interpreterSettingManager, JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials, NoteEventListener noteEventListener) { this.repo = repo; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; this.jobListenerFactory = jlFactory; this.index = noteIndex; this.noteEventListener = noteEventListener; @@ -117,7 +120,7 @@ public class Note implements Serializable, ParagraphJobListener { } private String getDefaultInterpreterName() { - InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId()); + InterpreterSetting setting = interpreterSettingManager.getDefaultInterpreterSetting(getId()); return null != setting ? setting.getName() : StringUtils.EMPTY; } @@ -220,6 +223,15 @@ public class Note implements Serializable, ParagraphJobListener { } } + void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) { + this.interpreterSettingManager = interpreterSettingManager; + synchronized (paragraphs) { + for (Paragraph p : paragraphs) { + p.setInterpreterSettingManager(interpreterSettingManager); + } + } + } + public void initializeJobListenerForParagraph(Paragraph paragraph) { final Note paragraphNote = paragraph.getNote(); if (!paragraphNote.getId().equals(this.getId())) { @@ -272,7 +284,7 @@ public class Note implements Serializable, ParagraphJobListener { * Add paragraph last. */ public Paragraph addParagraph(AuthenticationInfo authenticationInfo) { - Paragraph p = new Paragraph(this, this, factory); + Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager); p.setAuthenticationInfo(authenticationInfo); setParagraphMagic(p, paragraphs.size()); synchronized (paragraphs) { @@ -292,7 +304,8 @@ public class Note implements Serializable, ParagraphJobListener { void addCloneParagraph(Paragraph srcParagraph) { // Keep paragraph original ID - final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory); + final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory, + interpreterSettingManager); Map<String, Object> config = new HashMap<>(srcParagraph.getConfig()); Map<String, Object> param = new HashMap<>(srcParagraph.settings.getParams()); @@ -329,7 +342,7 @@ public class Note implements Serializable, ParagraphJobListener { * @param index index of paragraphs */ public Paragraph insertParagraph(int index, AuthenticationInfo authenticationInfo) { - Paragraph p = new Paragraph(this, this, factory); + Paragraph p = new Paragraph(this, this, factory, interpreterSettingManager); p.setAuthenticationInfo(authenticationInfo); setParagraphMagic(p, index); synchronized (paragraphs) { @@ -622,7 +635,7 @@ public class Note implements Serializable, ParagraphJobListener { private void snapshotAngularObjectRegistry(String user) { angularObjects = new HashMap<>(); - List<InterpreterSetting> settings = factory.getInterpreterSettings(getId()); + List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } @@ -637,7 +650,7 @@ public class Note implements Serializable, ParagraphJobListener { private void removeAllAngularObjectInParagraph(String user, String paragraphId) { angularObjects = new HashMap<>(); - List<InterpreterSetting> settings = factory.getInterpreterSettings(getId()); + List<InterpreterSetting> settings = interpreterSettingManager.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { return; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 8b946f2..474f98c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -77,6 +77,7 @@ public class Notebook implements NoteEventListener { private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; + private InterpreterSettingManager interpreterSettingManager; /** * Keep the order. */ @@ -102,13 +103,14 @@ public class Notebook implements NoteEventListener { */ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, SchedulerFactory schedulerFactory, InterpreterFactory replFactory, - JobListenerFactory jobListenerFactory, SearchService noteSearchService, - NotebookAuthorization notebookAuthorization, Credentials credentials) - throws IOException, SchedulerException { + InterpreterSettingManager interpreterSettingManager, JobListenerFactory jobListenerFactory, + SearchService noteSearchService, NotebookAuthorization notebookAuthorization, + Credentials credentials) throws IOException, SchedulerException { this.conf = conf; this.notebookRepo = notebookRepo; this.schedulerFactory = schedulerFactory; this.replFactory = replFactory; + this.interpreterSettingManager = interpreterSettingManager; this.jobListenerFactory = jobListenerFactory; this.noteSearchService = noteSearchService; this.notebookAuthorization = notebookAuthorization; @@ -138,7 +140,7 @@ public class Notebook implements NoteEventListener { Preconditions.checkNotNull(subject, "AuthenticationInfo should not be null"); Note note; if (conf.getBoolean(ConfVars.ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING)) { - note = createNote(replFactory.getDefaultInterpreterSettingList(), subject); + note = createNote(interpreterSettingManager.getDefaultInterpreterSettingList(), subject); } else { note = createNote(null, subject); } @@ -154,7 +156,7 @@ public class Notebook implements NoteEventListener { public Note createNote(List<String> interpreterIds, AuthenticationInfo subject) throws IOException { Note note = - new Note(notebookRepo, replFactory, jobListenerFactory, + new Note(notebookRepo, replFactory, interpreterSettingManager, jobListenerFactory, noteSearchService, credentials, this); note.setNoteNameListener(folders); @@ -270,14 +272,15 @@ public class Notebook implements NoteEventListener { throws IOException { Note note = getNote(id); if (note != null) { - List<InterpreterSetting> currentBindings = replFactory.getInterpreterSettings(id); + List<InterpreterSetting> currentBindings = + interpreterSettingManager.getInterpreterSettings(id); for (InterpreterSetting setting : currentBindings) { if (!interpreterSettingIds.contains(setting.getId())) { fireUnbindInterpreter(note, setting); } } - replFactory.setInterpreters(user, note.getId(), interpreterSettingIds); + interpreterSettingManager.setInterpreters(user, note.getId(), interpreterSettingIds); // comment out while note.getNoteReplLoader().setInterpreters(...) do the same // replFactory.putNoteInterpreterSettingBinding(id, interpreterSettingIds); } @@ -286,7 +289,7 @@ public class Notebook implements NoteEventListener { List<String> getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { - return getInterpreterFactory().getInterpreters(note.getId()); + return interpreterSettingManager.getInterpreters(note.getId()); } else { return new LinkedList<>(); } @@ -295,7 +298,7 @@ public class Notebook implements NoteEventListener { public List<InterpreterSetting> getBindedInterpreterSettings(String id) { Note note = getNote(id); if (note != null) { - return replFactory.getInterpreterSettings(note.getId()); + return interpreterSettingManager.getInterpreterSettings(note.getId()); } else { return new LinkedList<>(); } @@ -328,12 +331,12 @@ public class Notebook implements NoteEventListener { note = notes.remove(id); folders.removeNote(note); } - replFactory.removeNoteInterpreterSettingBinding(subject.getUser(), id); + interpreterSettingManager.removeNoteInterpreterSettingBinding(subject.getUser(), id); noteSearchService.deleteIndexDocs(note); notebookAuthorization.removeNote(id); // remove from all interpreter instance's angular object registry - for (InterpreterSetting settings : replFactory.get()) { + for (InterpreterSetting settings : interpreterSettingManager.get()) { AngularObjectRegistry registry = settings.getInterpreterGroup(subject.getUser(), id).getAngularObjectRegistry(); if (registry instanceof RemoteAngularObjectRegistry) { @@ -467,6 +470,7 @@ public class Notebook implements NoteEventListener { note.setCredentials(this.credentials); note.setInterpreterFactory(replFactory); + note.setInterpreterSettingManager(interpreterSettingManager); note.setJobListenerFactory(jobListenerFactory); note.setNotebookRepo(notebookRepo); @@ -509,7 +513,7 @@ public class Notebook implements NoteEventListener { for (String name : angularObjectSnapshot.keySet()) { SnapshotAngularObject snapshot = angularObjectSnapshot.get(name); - List<InterpreterSetting> settings = replFactory.get(); + List<InterpreterSetting> settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { InterpreterGroup intpGroup = setting.getInterpreterGroup(subject.getUser(), note.getId()); if (intpGroup.getId().equals(snapshot.getIntpGroupId())) { @@ -753,9 +757,10 @@ public class Notebook implements NoteEventListener { // set interpreter bind type String interpreterGroupName = null; - if (replFactory.getInterpreterSettings(jobNote.getId()) != null - && replFactory.getInterpreterSettings(jobNote.getId()).size() >= 1) { - interpreterGroupName = replFactory.getInterpreterSettings(jobNote.getId()).get(0).getName(); + if (interpreterSettingManager.getInterpreterSettings(jobNote.getId()) != null + && interpreterSettingManager.getInterpreterSettings(jobNote.getId()).size() >= 1) { + interpreterGroupName = + interpreterSettingManager.getInterpreterSettings(jobNote.getId()).get(0).getName(); } // note json object root information. @@ -829,9 +834,10 @@ public class Notebook implements NoteEventListener { // set interpreter bind type String interpreterGroupName = null; - if (replFactory.getInterpreterSettings(note.getId()) != null - && replFactory.getInterpreterSettings(note.getId()).size() >= 1) { - interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName(); + if (interpreterSettingManager.getInterpreterSettings(note.getId()) != null + && interpreterSettingManager.getInterpreterSettings(note.getId()).size() >= 1) { + interpreterGroupName = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0).getName(); } // not update and not running -> pass @@ -881,9 +887,9 @@ public class Notebook implements NoteEventListener { logger.error(e.getMessage(), e); } if (releaseResource) { - for (InterpreterSetting setting : notebook.getInterpreterFactory() + for (InterpreterSetting setting : notebook.getInterpreterSettingManager() .getInterpreterSettings(note.getId())) { - notebook.getInterpreterFactory().restart(setting.getId()); + notebook.getInterpreterSettingManager().restart(setting.getId()); } } } @@ -948,6 +954,10 @@ public class Notebook implements NoteEventListener { return replFactory; } + public InterpreterSettingManager getInterpreterSettingManager() { + return interpreterSettingManager; + } + public NotebookAuthorization getNotebookAuthorization() { return notebookAuthorization; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java index 28b6ab3..f609ecb 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Paragraph.java @@ -53,6 +53,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { private static Logger logger = LoggerFactory.getLogger(Paragraph.class); private transient InterpreterFactory factory; + private transient InterpreterSettingManager interpreterSettingManager; private transient Note note; private transient AuthenticationInfo authenticationInfo; private transient Map<String, Paragraph> userParagraphMap = Maps.newHashMap(); // personalized @@ -84,10 +85,11 @@ public class Paragraph extends Job implements Serializable, Cloneable { } public Paragraph(String paragraphId, Note note, JobListener listener, - InterpreterFactory factory) { + InterpreterFactory factory, InterpreterSettingManager interpreterSettingManager) { super(paragraphId, generateId(), listener); this.note = note; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; title = null; text = null; authenticationInfo = null; @@ -97,10 +99,12 @@ public class Paragraph extends Job implements Serializable, Cloneable { config = new HashMap<>(); } - public Paragraph(Note note, JobListener listener, InterpreterFactory factory) { + public Paragraph(Note note, JobListener listener, InterpreterFactory factory, + InterpreterSettingManager interpreterSettingManager) { super(generateId(), listener); this.note = note; this.factory = factory; + this.interpreterSettingManager = interpreterSettingManager; title = null; text = null; authenticationInfo = null; @@ -249,7 +253,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { public List<InterpreterCompletion> getInterpreterCompletion() { List<InterpreterCompletion> completion = new LinkedList(); - for (InterpreterSetting intp : factory.getInterpreterSettings(note.getId())) { + for (InterpreterSetting intp : interpreterSettingManager.getInterpreterSettings(note.getId())) { List<InterpreterInfo> intInfo = intp.getInterpreterInfos(); if (intInfo.size() > 1) { for (InterpreterInfo info : intInfo) { @@ -292,6 +296,10 @@ public class Paragraph extends Job implements Serializable, Cloneable { this.factory = factory; } + public void setInterpreterSettingManager(InterpreterSettingManager interpreterSettingManager) { + this.interpreterSettingManager = interpreterSettingManager; + } + public InterpreterResult getResult() { return (InterpreterResult) getReturn(); } @@ -416,7 +424,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { } private boolean noteHasInterpreters() { - return !factory.getInterpreterSettings(note.getId()).isEmpty(); + return !interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty(); } private boolean interpreterHasUser(InterpreterSetting intp) { @@ -430,7 +438,7 @@ public class Paragraph extends Job implements Serializable, Cloneable { private InterpreterSetting getInterpreterSettingById(String id) { InterpreterSetting setting = null; - for (InterpreterSetting i : factory.getInterpreterSettings(note.getId())) { + for (InterpreterSetting i : interpreterSettingManager.getInterpreterSettings(note.getId())) { if (id.startsWith(i.getId())) { setting = i; break; @@ -504,8 +512,9 @@ public class Paragraph extends Job implements Serializable, Cloneable { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; - if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { - InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); + if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { + InterpreterSetting intpGroup = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } @@ -532,8 +541,9 @@ public class Paragraph extends Job implements Serializable, Cloneable { AngularObjectRegistry registry = null; ResourcePool resourcePool = null; - if (!factory.getInterpreterSettings(note.getId()).isEmpty()) { - InterpreterSetting intpGroup = factory.getInterpreterSettings(note.getId()).get(0); + if (!interpreterSettingManager.getInterpreterSettings(note.getId()).isEmpty()) { + InterpreterSetting intpGroup = + interpreterSettingManager.getInterpreterSettings(note.getId()).get(0); registry = intpGroup.getInterpreterGroup(getUser(), note.getId()).getAngularObjectRegistry(); resourcePool = intpGroup.getInterpreterGroup(getUser(), note.getId()).getResourcePool(); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 2588c4c..3940fc3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -49,6 +49,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { private SchedulerFactory schedulerFactory; private DependencyResolver depResolver; private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private VFSNotebookRepo notebookRepo; private Notebook notebook; private HeliumApplicationFactory heliumAppFactory; @@ -84,8 +85,8 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { heliumAppFactory = new HeliumApplicationFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, - new InterpreterOption(true), null, null, heliumAppFactory, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, heliumAppFactory, depResolver, false, interpreterSettingManager); HashMap<String, String> env = new HashMap<>(); env.put("ZEPPELIN_CLASSPATH", new File("./target/test-classes").getAbsolutePath()); factory.setEnv(env); @@ -98,6 +99,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { notebookRepo, schedulerFactory, factory, + interpreterSettingManager, this, search, notebookAuthorization, @@ -112,7 +114,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { @After public void tearDown() throws Exception { - List<InterpreterSetting> settings = factory.get(); + List<InterpreterSetting> settings = interpreterSettingManager.get(); for (InterpreterSetting setting : settings) { for (InterpreterGroup intpGroup : setting.getAllInterpreterGroups()) { intpGroup.close(); @@ -138,7 +140,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { "", ""); Note note1 = notebook.createNote(anonymous); - factory.setInterpreters("user", note1.getId(),factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note1.getId(),interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -184,7 +186,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { "", ""); Note note1 = notebook.createNote(anonymous); - factory.setInterpreters("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + interpreterSettingManager.setInterpreters("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -224,7 +226,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { "", ""); Note note1 = notebook.createNote(anonymous); - notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); Paragraph p1 = note1.addParagraph(AuthenticationInfo.ANONYMOUS); @@ -285,7 +287,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { "", ""); Note note1 = notebook.createNote(anonymous); - notebook.bindInterpretersToNote("user", note1.getId(), factory.getDefaultInterpreterSettingList()); + notebook.bindInterpretersToNote("user", note1.getId(), interpreterSettingManager.getDefaultInterpreterSettingList()); String mock1IntpSettingId = null; for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.getId())) { if (setting.getName().equals("mock1")) { @@ -312,7 +314,7 @@ public class HeliumApplicationFactoryTest implements JobListenerFactory { Thread.yield(); } // when restart interpreter - factory.restart(mock1IntpSettingId); + interpreterSettingManager.restart(mock1IntpSettingId); while (app.getStatus() == ApplicationState.Status.LOADED) { Thread.yield(); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 7522366..a8da7d3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -68,6 +68,7 @@ import org.mockito.Mock; public class InterpreterFactoryTest { private InterpreterFactory factory; + private InterpreterSettingManager interpreterSettingManager; private File tmpDir; private ZeppelinConfiguration conf; private InterpreterContext context; @@ -102,13 +103,14 @@ public class InterpreterFactoryTest { conf = new ZeppelinConfiguration(); schedulerFactory = new SchedulerFactory(); depResolver = new DependencyResolver(tmpDir.getAbsolutePath() + "/local-repo"); - factory = new InterpreterFactory(conf, new InterpreterOption(false), null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); context = new InterpreterContext("note", "id", null, "title", "text", null, null, null, null, null, null, null); SearchService search = mock(SearchService.class); notebookRepo = new VFSNotebookRepo(conf); notebookAuthorization = NotebookAuthorization.init(conf); - notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, jobListenerFactory, search, + notebook = new Notebook(conf, notebookRepo, schedulerFactory, factory, interpreterSettingManager, jobListenerFactory, search, notebookAuthorization, null); } @@ -119,7 +121,7 @@ public class InterpreterFactoryTest { @Test public void testBasic() { - List<InterpreterSetting> all = factory.get(); + List<InterpreterSetting> all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -137,17 +139,18 @@ public class InterpreterFactoryTest { assertNotNull("get Interpreter", interpreterGroup.get("session").get(0)); // try to get unavailable interpreter - assertNull(factory.get("unknown")); + assertNull(interpreterSettingManager.get("unknown")); // restart interpreter - factory.restart(mock1Setting.getId()); + interpreterSettingManager.restart(mock1Setting.getId()); assertNull(mock1Setting.getInterpreterGroup("user", "sharedProcess").get("session")); } @Test public void testRemoteRepl() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List<InterpreterSetting> all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List<InterpreterSetting> all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -174,8 +177,9 @@ public class InterpreterFactoryTest { */ @Test public void testRestartInterpreterInScopedMode() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List<InterpreterSetting> all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List<InterpreterSetting> all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -210,8 +214,9 @@ public class InterpreterFactoryTest { */ @Test public void testRestartInterpreterInIsolatedMode() throws Exception { - factory = new InterpreterFactory(conf, new InterpreterOption(true), null, null, null, depResolver, false); - List<InterpreterSetting> all = factory.get(); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); + List<InterpreterSetting> all = interpreterSettingManager.get(); InterpreterSetting mock1Setting = null; for (InterpreterSetting setting : all) { if (setting.getName().equals("mock1")) { @@ -243,21 +248,21 @@ public class InterpreterFactoryTest { @Test public void testFactoryDefaultList() throws IOException, RepositoryException { // get default settings - List<String> all = factory.getDefaultInterpreterSettingList(); - assertTrue(factory.get().size() >= all.size()); + List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList(); + assertTrue(interpreterSettingManager.get().size() >= all.size()); } @Test public void testExceptions() throws InterpreterException, IOException, RepositoryException { - List<String> all = factory.getDefaultInterpreterSettingList(); + List<String> all = interpreterSettingManager.getDefaultInterpreterSettingList(); // add setting with null option & properties expected nullArgumentException.class try { - factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); + interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); } catch(NullArgumentException e) { assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage()); } try { - factory.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); + interpreterSettingManager.add("mock2", new ArrayList<InterpreterInfo>(), new LinkedList<Dependency>(), new InterpreterOption(false), Collections.EMPTY_MAP, "", null); } catch (NullArgumentException e){ assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage()); } @@ -267,22 +272,23 @@ public class InterpreterFactoryTest { @Test public void testSaveLoad() throws IOException, RepositoryException { // interpreter settings - int numInterpreters = factory.get().size(); + int numInterpreters = interpreterSettingManager.get().size(); // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - factory.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); - assertEquals(numInterpreters + 1, factory.get().size()); + interpreterSettingManager.createNewSetting("new-mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); + assertEquals(numInterpreters + 1, interpreterSettingManager.get().size()); - InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver, false); - assertEquals(numInterpreters + 1, factory2.get().size()); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + + assertEquals(numInterpreters + 1, interpreterSettingManager.get().size()); } @Test public void testInterpreterSettingPropertyClass() throws IOException, RepositoryException { // check if default interpreter reference's property type is map - Map<String, InterpreterSetting> interpreterSettingRefs = factory.getAvailableInterpreterSettings(); + Map<String, InterpreterSetting> interpreterSettingRefs = interpreterSettingManager.getAvailableInterpreterSettings(); InterpreterSetting intpSetting = interpreterSettingRefs.get("mock1"); Map<String, InterpreterProperty> intpProperties = (Map<String, InterpreterProperty>) intpSetting.getProperties(); @@ -293,7 +299,7 @@ public class InterpreterFactoryTest { properties.put("key1", "value1"); properties.put("key2", "value2"); - factory.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties); + interpreterSettingManager.createNewSetting("newMock", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), properties); String confFilePath = conf.getInterpreterSettingPath(); byte[] encoded = Files.readAllBytes(Paths.get(confFilePath)); @@ -312,20 +318,21 @@ public class InterpreterFactoryTest { @Test public void testInterpreterAliases() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver, false); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, false, interpreterSettingManager); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true, null); - factory.add("group1", new ArrayList<InterpreterInfo>() {{ + interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>() {{ add(info1); }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null); - factory.add("group2", new ArrayList<InterpreterInfo>(){{ + interpreterSettingManager.add("group2", new ArrayList<InterpreterInfo>(){{ add(info2); }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path2", null); - final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties()); - final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties()); + final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties()); + final InterpreterSetting setting2 = interpreterSettingManager.createNewSetting("test-group2", "group1", new ArrayList<Dependency>(), new InterpreterOption(true), new Properties()); - factory.setInterpreters("user", "note", new ArrayList<String>() {{ + interpreterSettingManager.setInterpreters("user", "note", new ArrayList<String>() {{ add(setting1.getId()); add(setting2.getId()); }}); @@ -336,20 +343,21 @@ public class InterpreterFactoryTest { @Test public void testMultiUser() throws IOException, RepositoryException { - factory = new InterpreterFactory(conf, null, null, null, depResolver, true); + interpreterSettingManager = new InterpreterSettingManager(conf, depResolver, new InterpreterOption(true)); + factory = new InterpreterFactory(conf, null, null, null, depResolver, true, interpreterSettingManager); final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true, null); - factory.add("group1", new ArrayList<InterpreterInfo>(){{ + interpreterSettingManager.add("group1", new ArrayList<InterpreterInfo>(){{ add(info1); }}, new ArrayList<Dependency>(), new InterpreterOption(true), Collections.EMPTY_MAP, "/path1", null); InterpreterOption perUserInterpreterOption = new InterpreterOption(true, InterpreterOption.ISOLATED, InterpreterOption.SHARED); - final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties()); + final InterpreterSetting setting1 = interpreterSettingManager.createNewSetting("test-group1", "group1", new ArrayList<Dependency>(), perUserInterpreterOption, new Properties()); - factory.setInterpreters("user1", "note", new ArrayList<String>() {{ + interpreterSettingManager.setInterpreters("user1", "note", new ArrayList<String>() {{ add(setting1.getId()); }}); - factory.setInterpreters("user2", "note", new ArrayList<String>() {{ + interpreterSettingManager.setInterpreters("user2", "note", new ArrayList<String>() {{ add(setting1.getId()); }}); @@ -360,7 +368,7 @@ public class InterpreterFactoryTest { @Test public void testInvalidInterpreterSettingName() { try { - factory.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); + interpreterSettingManager.createNewSetting("new.mock1", "mock1", new LinkedList<Dependency>(), new InterpreterOption(false), new Properties()); fail("expect fail because of invalid InterpreterSetting Name"); } catch (IOException e) { assertEquals("'.' is invalid for InterpreterSetting name.", e.getMessage()); @@ -371,39 +379,40 @@ public class InterpreterFactoryTest { @Test public void getEditorSetting() throws IOException, RepositoryException, SchedulerException { List<String> intpIds = new ArrayList<>(); - for(InterpreterSetting intpSetting: factory.get()) { + for(InterpreterSetting intpSetting: interpreterSettingManager.get()) { if (intpSetting.getName().startsWith("mock1")) { intpIds.add(intpSetting.getId()); } } Note note = notebook.createNote(intpIds, new AuthenticationInfo("anonymous")); + Interpreter interpreter = factory.getInterpreter("user1", note.getId(), "mock11"); // get editor setting from interpreter-setting.json - Map<String, Object> editor = factory.getEditorSetting("user1", note.getId(), "mock11"); + Map<String, Object> editor = interpreterSettingManager.getEditorSetting(interpreter, "user1", note.getId(), "mock11"); assertEquals("java", editor.get("language")); // when interpreter is not loaded via interpreter-setting.json // or editor setting doesn't exit - editor = factory.getEditorSetting("user1", note.getId(), "mock1"); + editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock1"),"user1", note.getId(), "mock1"); assertEquals(null, editor.get("language")); // when interpreter is not bound to note - editor = factory.getEditorSetting("user1", note.getId(), "mock2"); + editor = interpreterSettingManager.getEditorSetting(factory.getInterpreter("user1", note.getId(), "mock11"),"user1", note.getId(), "mock2"); assertEquals("text", editor.get("language")); } @Test public void registerCustomInterpreterRunner() throws IOException { - InterpreterFactory spyFactory = spy(factory); + InterpreterSettingManager spyInterpreterSettingManager = spy(interpreterSettingManager); - doNothing().when(spyFactory).saveToFile(); + doNothing().when(spyInterpreterSettingManager).saveToFile(); ArrayList<InterpreterInfo> interpreterInfos1 = new ArrayList<>(); interpreterInfos1.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap())); - spyFactory.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null); + spyInterpreterSettingManager.add("normalGroup1", interpreterInfos1, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/normalGroup1", null); - spyFactory.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties()); + spyInterpreterSettingManager.createNewSetting("normalGroup1", "normalGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties()); ArrayList<InterpreterInfo> interpreterInfos2 = new ArrayList<>(); interpreterInfos2.add(new InterpreterInfo("name1.class", "name1", true, Maps.<String, Object>newHashMap())); @@ -412,13 +421,13 @@ public class InterpreterFactoryTest { when(mockInterpreterRunner.getPath()).thenReturn("custom-linux-path.sh"); - spyFactory.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner); + spyInterpreterSettingManager.add("customGroup1", interpreterInfos2, Lists.<Dependency>newArrayList(), new InterpreterOption(true), Maps.<String, InterpreterProperty>newHashMap(), "/customGroup1", mockInterpreterRunner); - spyFactory.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties()); + spyInterpreterSettingManager.createNewSetting("customGroup1", "customGroup1", Lists.<Dependency>newArrayList(), new InterpreterOption(true), new Properties()); - spyFactory.setInterpreters("anonymous", "noteCustome", spyFactory.getDefaultInterpreterSettingList()); + spyInterpreterSettingManager.setInterpreters("anonymous", "noteCustome", spyInterpreterSettingManager.getDefaultInterpreterSettingList()); - spyFactory.getInterpreter("anonymous", "noteCustome", "customGroup1"); + factory.getInterpreter("anonymous", "noteCustome", "customGroup1"); verify(mockInterpreterRunner, times(1)).getPath(); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/176a37f3/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java index 8325e8a..27aa633 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/FolderTest.java @@ -19,6 +19,7 @@ package org.apache.zeppelin.notebook; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterSettingManager; import org.apache.zeppelin.notebook.repo.NotebookRepo; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.search.SearchService; @@ -59,6 +60,9 @@ public class FolderTest { @Mock InterpreterFactory interpreterFactory; + @Mock + InterpreterSettingManager interpreterSettingManager; + Folder folder; Note note1; @@ -67,13 +71,13 @@ public class FolderTest { @Before public void createFolderAndNotes() { - note1 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note1 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note1.setName("this/is/a/folder/note1"); - note2 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note2 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note2.setName("this/is/a/folder/note2"); - note3 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + note3 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note3.setName("this/is/a/folder/note3"); folder = new Folder("this/is/a/folder"); @@ -114,7 +118,7 @@ public class FolderTest { @Test public void addNoteTest() { - Note note4 = new Note(repo, interpreterFactory, jobListenerFactory, index, credentials, noteEventListener); + Note note4 = new Note(repo, interpreterFactory, interpreterSettingManager, jobListenerFactory, index, credentials, noteEventListener); note4.setName("this/is/a/folder/note4"); folder.addNote(note4);
