http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java index 7d1df9b..d76bb15 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java @@ -17,13 +17,12 @@ package org.apache.zeppelin.interpreter; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.IOException; import java.io.StringReader; import java.util.Properties; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Special Interpreter for Interpreter Configuration customization. It is attached to each @@ -37,11 +36,11 @@ public class ConfInterpreter extends Interpreter { protected String interpreterGroupId; protected InterpreterSetting interpreterSetting; - - public ConfInterpreter(Properties properties, - String sessionId, - String interpreterGroupId, - InterpreterSetting interpreterSetting) { + public ConfInterpreter( + Properties properties, + String sessionId, + String interpreterGroupId, + InterpreterSetting interpreterSetting) { super(properties); this.sessionId = sessionId; this.interpreterGroupId = interpreterGroupId; @@ -49,14 +48,10 @@ public class ConfInterpreter extends Interpreter { } @Override - public void open() throws InterpreterException { - - } + public void open() throws InterpreterException {} @Override - public void close() throws InterpreterException { - - } + public void close() throws InterpreterException {} @Override public InterpreterResult interpret(String st, InterpreterContext context) @@ -68,8 +63,8 @@ public class ConfInterpreter extends Interpreter { Properties newProperties = new Properties(); newProperties.load(new StringReader(st)); finalProperties.putAll(newProperties); - LOGGER.debug("Properties for InterpreterGroup: " + interpreterGroupId + " is " - + finalProperties); + LOGGER.debug( + "Properties for InterpreterGroup: " + interpreterGroupId + " is " + finalProperties); interpreterSetting.setInterpreterGroupProperties(interpreterGroupId, finalProperties); return new InterpreterResult(InterpreterResult.Code.SUCCESS); } catch (IOException e) { @@ -79,9 +74,7 @@ public class ConfInterpreter extends Interpreter { } @Override - public void cancel(InterpreterContext context) throws InterpreterException { - - } + public void cancel(InterpreterContext context) throws InterpreterException {} @Override public FormType getFormType() throws InterpreterException {
http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index e045a59..be8e961 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -17,18 +17,14 @@ package org.apache.zeppelin.interpreter; -import com.google.common.base.Preconditions; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; - /** * //TODO(zjffdu) considering to move to InterpreterSettingManager * - * Factory class for creating interpreters. - * + * <p>Factory class for creating interpreters. */ public class InterpreterFactory { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterFactory.class); @@ -39,10 +35,8 @@ public class InterpreterFactory { this.interpreterSettingManager = interpreterSettingManager; } - public Interpreter getInterpreter(String user, - String noteId, - String replName, - String defaultInterpreterSetting) + public Interpreter getInterpreter( + String user, String noteId, String replName, String defaultInterpreterSetting) throws InterpreterNotFoundException { if (StringUtils.isBlank(replName)) { @@ -66,10 +60,9 @@ public class InterpreterFactory { } throw new InterpreterNotFoundException("No interpreter setting named: " + group); - } else if (replNameSplits.length == 1){ + } else if (replNameSplits.length == 1) { // first assume group is omitted - InterpreterSetting setting = - interpreterSettingManager.getByName(defaultInterpreterSetting); + InterpreterSetting setting = interpreterSettingManager.getByName(defaultInterpreterSetting); if (setting != null) { Interpreter interpreter = setting.getInterpreter(user, noteId, replName); if (null != interpreter) { http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java index fd632ce..a721ea0 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java @@ -18,21 +18,23 @@ package org.apache.zeppelin.interpreter; import com.google.gson.annotations.SerializedName; - import java.util.Map; /** - * Information of interpreters in this interpreter setting. - * this will be serialized for conf/interpreter.json and REST api response. + * Information of interpreters in this interpreter setting. this will be serialized for + * conf/interpreter.json and REST api response. */ public class InterpreterInfo { private String name; - @SerializedName("class") private String className; + + @SerializedName("class") + private String className; + private boolean defaultInterpreter = false; private Map<String, Object> editor; - public InterpreterInfo(String className, String name, boolean defaultInterpreter, - Map<String, Object> editor) { + public InterpreterInfo( + String className, String name, boolean defaultInterpreter, Map<String, Object> editor) { this.className = className; this.name = name; this.defaultInterpreter = defaultInterpreter; @@ -72,9 +74,10 @@ public class InterpreterInfo { boolean sameName = null == getName() ? null == other.getName() : getName().equals(other.getName()); - boolean sameClassName = null == getClassName() ? - null == other.getClassName() : - getClassName().equals(other.getClassName()); + boolean sameClassName = + null == getClassName() + ? null == other.getClassName() + : getClassName().equals(other.getClassName()); boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter(); return sameName && sameClassName && sameIsDefaultInterpreter; http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java index 8f89448..22eb66c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfoSaving.java @@ -17,37 +17,32 @@ package org.apache.zeppelin.interpreter; +import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; +import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; + import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import com.google.gson.internal.StringMap; -import org.apache.commons.io.IOUtils; -import org.apache.zeppelin.common.JsonSerializable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.repository.RemoteRepository; - import java.io.BufferedReader; import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.PosixFilePermission; import java.util.*; +import org.apache.commons.io.IOUtils; +import org.apache.zeppelin.common.JsonSerializable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.repository.RemoteRepository; -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ; -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE; - -/** - * - */ +/** */ public class InterpreterInfoSaving implements JsonSerializable { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterInfoSaving.class); - private static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); + private static final Gson gson = new GsonBuilder().setPrettyPrinting().create(); public Map<String, InterpreterSetting> interpreterSettings = new HashMap<>(); public List<RemoteRepository> interpreterRepositories = new ArrayList<>(); @@ -63,7 +58,8 @@ public class InterpreterInfoSaving implements JsonSerializable { if (infoSaving != null && infoSaving.interpreterSettings != null) { for (InterpreterSetting interpreterSetting : infoSaving.interpreterSettings.values()) { interpreterSetting.convertPermissionsFromUsersToOwners( - jsonObject.getAsJsonObject("interpreterSettings") + jsonObject + .getAsJsonObject("interpreterSettings") .getAsJsonObject(interpreterSetting.getId())); } } @@ -80,7 +76,8 @@ public class InterpreterInfoSaving implements JsonSerializable { } catch (UnsupportedOperationException e) { // File system does not support Posix file permissions (likely windows) - continue anyway. LOGGER.warn("unable to setPosixFilePermissions on '{}'.", file); - }; + } + ; } LOGGER.info("Save Interpreter Settings to " + file); IOUtils.write(this.toJson(), new FileOutputStream(file.toFile())); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java index 192e822..15b5fdb 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterNotFoundException.java @@ -1,12 +1,9 @@ package org.apache.zeppelin.interpreter; -/** - * Exception for no interpreter is found - */ +/** Exception for no interpreter is found */ public class InterpreterNotFoundException extends InterpreterException { - public InterpreterNotFoundException() { - } + public InterpreterNotFoundException() {} public InterpreterNotFoundException(String message) { super(message); http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index 36fc1f1..335503c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -17,6 +17,10 @@ package org.apache.zeppelin.interpreter; +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; +import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; +import static org.apache.zeppelin.util.IdHashes.generateId; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -25,6 +29,19 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.annotations.SerializedName; import com.google.gson.internal.StringMap; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.commons.io.FileUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.dep.Dependency; @@ -45,51 +62,29 @@ import org.apache.zeppelin.plugin.PluginManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE; -import static org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; -import static org.apache.zeppelin.util.IdHashes.generateId; - -/** - * Represent one InterpreterSetting in the interpreter setting page - */ +/** Represent one InterpreterSetting in the interpreter setting page */ public class InterpreterSetting { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSetting.class); private static final String SHARED_PROCESS = "shared_process"; private static final String SHARED_SESSION = "shared_session"; - private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of( - "language", (Object) "text", - "editOnDblClick", false); + private static final Map<String, Object> DEFAULT_EDITOR = + ImmutableMap.of("language", (Object) "text", "editOnDblClick", false); private String id; private String name; // the original interpreter setting template name where it is created from private String group; - //TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json + // TODO(zjffdu) make the interpreter.json consistent with interpreter-setting.json /** - * properties can be either Properties or Map<String, InterpreterProperty> - * properties should be: - * - Properties when Interpreter instances are saved to `conf/interpreter.json` file - * - Map<String, InterpreterProperty> when Interpreters are registered - * : this is needed after https://github.com/apache/zeppelin/pull/1145 - * which changed the way of getting default interpreter setting AKA interpreterSettingsRef - * Note(mina): In order to simplify the implementation, I chose to change properties - * from Properties to Object instead of creating new classes. + * properties can be either Properties or Map<String, InterpreterProperty> properties should be: - + * Properties when Interpreter instances are saved to `conf/interpreter.json` file - Map<String, + * InterpreterProperty> when Interpreters are registered : this is needed after + * https://github.com/apache/zeppelin/pull/1145 which changed the way of getting default + * interpreter setting AKA interpreterSettingsRef Note(mina): In order to simplify the + * implementation, I chose to change properties from Properties to Object instead of creating new + * classes. */ private Object properties = new Properties(); @@ -130,9 +125,7 @@ public class InterpreterSetting { private transient RemoteInterpreterEventServer interpreterEventServer; /////////////////////////////////////////////////////////////////////////////////////////// - /** - * Builder class for InterpreterSetting - */ + /** Builder class for InterpreterSetting */ public static class Builder { private InterpreterSetting interpreterSetting; @@ -206,13 +199,14 @@ public class InterpreterSetting { return this; } - public Builder setRemoteInterpreterEventServer(RemoteInterpreterEventServer interpreterEventServer) { + public Builder setRemoteInterpreterEventServer( + RemoteInterpreterEventServer interpreterEventServer) { interpreterSetting.interpreterEventServer = interpreterEventServer; return this; } - public Builder setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener - remoteInterpreterProcessListener) { + public Builder setRemoteInterpreterProcessListener( + RemoteInterpreterProcessListener remoteInterpreterProcessListener) { interpreterSetting.remoteInterpreterProcessListener = remoteInterpreterProcessListener; return this; } @@ -277,8 +271,8 @@ public class InterpreterSetting { this.id = o.name; this.name = o.name; this.group = o.group; - this.properties = convertInterpreterProperties( - (Map<String, DefaultInterpreterProperty>) o.getProperties()); + this.properties = + convertInterpreterProperties((Map<String, DefaultInterpreterProperty>) o.getProperties()); this.interpreterInfos = new ArrayList<>(o.getInterpreterInfos()); this.option = InterpreterOption.fromInterpreterOption(o.getOption()); this.dependencies = new ArrayList<>(o.getDependencies()); @@ -288,8 +282,8 @@ public class InterpreterSetting { } private void createLauncher() throws IOException { - this.launcher = PluginManager.get().loadInterpreterLauncher( - getLauncherPlugin(), recoveryStorage); + this.launcher = + PluginManager.get().loadInterpreterLauncher(getLauncherPlugin(), recoveryStorage); } public AngularObjectRegistryListener getAngularObjectRegistryListener() { @@ -312,8 +306,8 @@ public class InterpreterSetting { return interpreterSettingManager; } - public InterpreterSetting setAngularObjectRegistryListener(AngularObjectRegistryListener - angularObjectRegistryListener) { + public InterpreterSetting setAngularObjectRegistryListener( + AngularObjectRegistryListener angularObjectRegistryListener) { this.angularObjectRegistryListener = angularObjectRegistryListener; return this; } @@ -323,8 +317,8 @@ public class InterpreterSetting { return this; } - public InterpreterSetting setRemoteInterpreterProcessListener(RemoteInterpreterProcessListener - remoteInterpreterProcessListener) { + public InterpreterSetting setRemoteInterpreterProcessListener( + RemoteInterpreterProcessListener remoteInterpreterProcessListener) { this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; return this; } @@ -386,7 +380,7 @@ public class InterpreterSetting { key = SHARED_PROCESS; } - //TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design + // TODO(zjffdu) we encode interpreter setting id into groupId, this is not a good design return id + ":" + key; } @@ -412,14 +406,18 @@ public class InterpreterSetting { try { interpreterGroupWriteLock.lock(); if (!interpreterGroups.containsKey(groupId)) { - LOGGER.info("Create InterpreterGroup with groupId: {} for user: {} and note: {}", - groupId, user, noteId); + LOGGER.info( + "Create InterpreterGroup with groupId: {} for user: {} and note: {}", + groupId, + user, + noteId); ManagedInterpreterGroup intpGroup = createInterpreterGroup(groupId); interpreterGroups.put(groupId, intpGroup); } return interpreterGroups.get(groupId); } finally { - interpreterGroupWriteLock.unlock();; + interpreterGroupWriteLock.unlock(); + ; } } @@ -438,7 +436,8 @@ public class InterpreterSetting { interpreterGroupReadLock.lock(); return interpreterGroups.get(groupId); } finally { - interpreterGroupReadLock.unlock();; + interpreterGroupReadLock.unlock(); + ; } } @@ -496,7 +495,6 @@ public class InterpreterSetting { } } - public Object getProperties() { return properties; } @@ -513,25 +511,25 @@ public class InterpreterSetting { Map<String, InterpreterProperty> iProperties = (Map<String, InterpreterProperty>) properties; for (Map.Entry<String, InterpreterProperty> entry : iProperties.entrySet()) { if (entry.getValue().getValue() != null) { - jProperties.setProperty(entry.getKey().trim(), - entry.getValue().getValue().toString().trim()); + jProperties.setProperty( + entry.getKey().trim(), entry.getValue().getValue().toString().trim()); } } if (!jProperties.containsKey("zeppelin.interpreter.output.limit")) { - jProperties.setProperty("zeppelin.interpreter.output.limit", - conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + ""); + jProperties.setProperty( + "zeppelin.interpreter.output.limit", conf.getInt(ZEPPELIN_INTERPRETER_OUTPUT_LIMIT) + ""); } if (!jProperties.containsKey("zeppelin.interpreter.max.poolsize")) { - jProperties.setProperty("zeppelin.interpreter.max.poolsize", + jProperties.setProperty( + "zeppelin.interpreter.max.poolsize", conf.getInt(ZEPPELIN_INTERPRETER_MAX_POOL_SIZE) + ""); } String interpreterLocalRepoPath = conf.getInterpreterLocalRepoPath(); - //TODO(zjffdu) change it to interpreterDir/{interpreter_name} - jProperties.setProperty("zeppelin.interpreter.localRepo", - interpreterLocalRepoPath + "/" + id); + // TODO(zjffdu) change it to interpreterDir/{interpreter_name} + jProperties.setProperty("zeppelin.interpreter.localRepo", interpreterLocalRepoPath + "/" + id); return jProperties; } @@ -598,9 +596,7 @@ public class InterpreterSetting { this.name = name; } - /*** - * Interpreter status - */ + /** * Interpreter status */ public enum Status { DOWNLOADING_DEPENDENCIES, ERROR, @@ -662,15 +658,19 @@ public class InterpreterSetting { List<InterpreterInfo> interpreterInfos = getInterpreterInfos(); Properties intpProperties = getJavaProperties(); for (InterpreterInfo info : interpreterInfos) { - Interpreter interpreter = new RemoteInterpreter(intpProperties, sessionId, - info.getClassName(), user, lifecycleManager); + Interpreter interpreter = + new RemoteInterpreter( + intpProperties, sessionId, info.getClassName(), user, lifecycleManager); if (info.isDefaultInterpreter()) { interpreters.add(0, interpreter); } else { interpreters.add(interpreter); } - LOGGER.info("Interpreter {} created for user: {}, sessionId: {}", - interpreter.getClassName(), user, sessionId); + LOGGER.info( + "Interpreter {} created for user: {}, sessionId: {}", + interpreter.getClassName(), + user, + sessionId); } // TODO(zjffdu) this kind of hardcode is ugly. For now SessionConfInterpreter is used @@ -685,16 +685,23 @@ public class InterpreterSetting { return interpreters; } - synchronized RemoteInterpreterProcess createInterpreterProcess(String interpreterGroupId, - String userName, - Properties properties) - throws IOException { + synchronized RemoteInterpreterProcess createInterpreterProcess( + String interpreterGroupId, String userName, Properties properties) throws IOException { if (launcher == null) { createLauncher(); } - InterpreterLaunchContext launchContext = new - InterpreterLaunchContext(properties, option, interpreterRunner, userName, - interpreterGroupId, id, group, name, interpreterEventServer.getPort(), interpreterEventServer.getHost()); + InterpreterLaunchContext launchContext = + new InterpreterLaunchContext( + properties, + option, + interpreterRunner, + userName, + interpreterGroupId, + id, + group, + name, + interpreterEventServer.getPort(), + interpreterEventServer.getHost()); RemoteInterpreterProcess process = (RemoteInterpreterProcess) launcher.launch(launchContext); recoveryStorage.onInterpreterClientStart(process); return process; @@ -702,8 +709,8 @@ public class InterpreterSetting { List<Interpreter> getOrCreateSession(String user, String noteId) { ManagedInterpreterGroup interpreterGroup = getOrCreateInterpreterGroup(user, noteId); - Preconditions.checkNotNull(interpreterGroup, "No InterpreterGroup existed for user {}, " + - "noteId {}", user, noteId); + Preconditions.checkNotNull( + interpreterGroup, "No InterpreterGroup existed for user {}, " + "noteId {}", user, noteId); String sessionId = getInterpreterSessionId(user, noteId); return interpreterGroup.getOrCreateSession(user, sessionId); } @@ -738,7 +745,7 @@ public class InterpreterSetting { return info.getClassName(); } } - //TODO(zjffdu) It requires user can not create interpreter with name `conf`, + // TODO(zjffdu) It requires user can not create interpreter with name `conf`, // conf is a reserved word of interpreter name if (replName.equals("conf")) { if (group.equals("livy")) { @@ -771,11 +778,12 @@ public class InterpreterSetting { ManagedInterpreterGroup interpreterGroup = this.interpreterGroups.get(interpreterGroupId); for (List<Interpreter> session : interpreterGroup.sessions.values()) { for (Interpreter intp : session) { - if (!intp.getProperties().equals(properties) && - interpreterGroup.getRemoteInterpreterProcess() != null && - interpreterGroup.getRemoteInterpreterProcess().isRunning()) { - throw new IOException("Can not change interpreter properties when interpreter process " + - "has already been launched"); + if (!intp.getProperties().equals(properties) + && interpreterGroup.getRemoteInterpreterProcess() != null + && interpreterGroup.getRemoteInterpreterProcess().isRunning()) { + throw new IOException( + "Can not change interpreter properties when interpreter process " + + "has already been launched"); } intp.setProperties(properties); } @@ -785,53 +793,58 @@ public class InterpreterSetting { private void loadInterpreterDependencies() { setStatus(Status.DOWNLOADING_DEPENDENCIES); setErrorReason(null); - Thread t = new Thread() { - public void run() { - try { - // dependencies to prevent library conflict - File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); - if (localRepoDir.exists()) { + Thread t = + new Thread() { + public void run() { try { - FileUtils.forceDelete(localRepoDir); - } catch (FileNotFoundException e) { - LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e); - } - } + // dependencies to prevent library conflict + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); + if (localRepoDir.exists()) { + try { + FileUtils.forceDelete(localRepoDir); + } catch (FileNotFoundException e) { + LOGGER.info("A file that does not exist cannot be deleted, nothing to worry", e); + } + } - // load dependencies - List<Dependency> deps = getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = new File( - conf.getRelativeDir(ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - if (d.getExclusions() != null) { - dependencyResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), - new File(destDir, id)); - } else { - dependencyResolver - .load(d.getGroupArtifactVersion(), new File(destDir, id)); + // load dependencies + List<Dependency> deps = getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = + new File( + conf.getRelativeDir( + ZeppelinConfiguration.ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + if (d.getExclusions() != null) { + dependencyResolver.load( + d.getGroupArtifactVersion(), d.getExclusions(), new File(destDir, id)); + } else { + dependencyResolver.load(d.getGroupArtifactVersion(), new File(destDir, id)); + } + } } + + setStatus(Status.READY); + setErrorReason(null); + } catch (Exception e) { + LOGGER.error( + String.format( + "Error while downloading repos for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly. : %s", + getGroup(), e.getLocalizedMessage()), + e); + setErrorReason(e.getLocalizedMessage()); + setStatus(Status.ERROR); } } - - setStatus(Status.READY); - setErrorReason(null); - } catch (Exception e) { - LOGGER.error(String.format("Error while downloading repos for interpreter group : %s," + - " go to interpreter setting page click on edit and save it again to make " + - "this interpreter work properly. : %s", - getGroup(), e.getLocalizedMessage()), e); - setErrorReason(e.getLocalizedMessage()); - setStatus(Status.ERROR); - } - } - }; + }; t.start(); } - //TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable + // TODO(zjffdu) ugly code, should not use JsonObject as parameter. not readable public void convertPermissionsFromUsersToOwners(JsonObject jsonObject) { if (jsonObject != null) { JsonObject option = jsonObject.getAsJsonObject("option"); @@ -857,10 +870,11 @@ public class InterpreterSetting { for (Object o : p.entrySet()) { Map.Entry entry = (Map.Entry) o; if (!(entry.getValue() instanceof StringMap)) { - InterpreterProperty newProperty = new InterpreterProperty( - entry.getKey().toString(), - entry.getValue(), - InterpreterPropertyType.STRING.getValue()); + InterpreterProperty newProperty = + new InterpreterProperty( + entry.getKey().toString(), + entry.getValue(), + InterpreterPropertyType.STRING.getValue()); newProperties.put(entry.getKey().toString(), newProperty); } else { // already converted @@ -870,8 +884,7 @@ public class InterpreterSetting { return newProperties; } else if (properties instanceof Map) { - Map<String, Object> dProperties = - (Map<String, Object>) properties; + Map<String, Object> dProperties = (Map<String, Object>) properties; Map<String, InterpreterProperty> newProperties = new HashMap<>(); for (String key : dProperties.keySet()) { Object value = dProperties.get(key); @@ -879,31 +892,29 @@ public class InterpreterSetting { return (Map<String, InterpreterProperty>) properties; } else if (value instanceof StringMap) { StringMap stringMap = (StringMap) value; - InterpreterProperty newProperty = new InterpreterProperty( - key, - stringMap.get("value"), - stringMap.containsKey("type") ? stringMap.get("type").toString() : "string"); + InterpreterProperty newProperty = + new InterpreterProperty( + key, + stringMap.get("value"), + stringMap.containsKey("type") ? stringMap.get("type").toString() : "string"); newProperties.put(newProperty.getName(), newProperty); - } else if (value instanceof DefaultInterpreterProperty){ + } else if (value instanceof DefaultInterpreterProperty) { DefaultInterpreterProperty dProperty = (DefaultInterpreterProperty) value; - InterpreterProperty property = new InterpreterProperty( - key, - dProperty.getValue(), - dProperty.getType() != null ? dProperty.getType() : "string" - // in case user forget to specify type in interpreter-setting.json - ); + InterpreterProperty property = + new InterpreterProperty( + key, + dProperty.getValue(), + dProperty.getType() != null ? dProperty.getType() : "string" + // in case user forget to specify type in interpreter-setting.json + ); newProperties.put(key, property); } else if (value instanceof String) { - InterpreterProperty newProperty = new InterpreterProperty( - key, - value, - "string"); + InterpreterProperty newProperty = new InterpreterProperty(key, value, "string"); newProperties.put(newProperty.getName(), newProperty); } else { - throw new RuntimeException("Can not convert this type of property: " + - value.getClass()); + throw new RuntimeException("Can not convert this type of property: " + value.getClass()); } } return newProperties; @@ -912,8 +923,9 @@ public class InterpreterSetting { } public void waitForReady() throws InterruptedException { - while (getStatus().equals( - org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { + while (getStatus() + .equals( + org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES)) { Thread.sleep(200); } } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index d730db4..d96c59f 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -25,6 +25,26 @@ import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream.Filter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.Set; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; @@ -43,65 +63,38 @@ import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; -import org.apache.zeppelin.util.ReflectionUtils; import org.apache.zeppelin.storage.ConfigStorage; +import org.apache.zeppelin.util.ReflectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.sonatype.aether.repository.Authentication; import org.sonatype.aether.repository.Proxy; import org.sonatype.aether.repository.RemoteRepository; -import org.sonatype.aether.repository.Authentication; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.DirectoryStream.Filter; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - /** * InterpreterSettingManager is the component which manage all the interpreter settings. - * (load/create/update/remove/get) - * TODO(zjffdu) We could move it into another separated component. + * (load/create/update/remove/get) TODO(zjffdu) We could move it into another separated component. */ public class InterpreterSettingManager implements InterpreterSettingManagerMBean { private static final Logger LOGGER = LoggerFactory.getLogger(InterpreterSettingManager.class); - private static final Map<String, Object> DEFAULT_EDITOR = ImmutableMap.of( - "language", (Object) "text", - "editOnDblClick", false); + private static final Map<String, Object> DEFAULT_EDITOR = + ImmutableMap.of("language", (Object) "text", "editOnDblClick", false); private final ZeppelinConfiguration conf; private final Path interpreterDirPath; /** - * This is only InterpreterSetting templates with default name and properties - * name --> InterpreterSetting + * This is only InterpreterSetting templates with default name and properties name --> + * InterpreterSetting */ private final Map<String, InterpreterSetting> interpreterSettingTemplates = Maps.newConcurrentMap(); /** - * This is used by creating and running Interpreters - * id --> InterpreterSetting - * TODO(zjffdu) change it to name --> InterpreterSetting + * This is used by creating and running Interpreters id --> InterpreterSetting TODO(zjffdu) change + * it to name --> InterpreterSetting */ - private final Map<String, InterpreterSetting> interpreterSettings = - Maps.newConcurrentMap(); + private final Map<String, InterpreterSetting> interpreterSettings = Maps.newConcurrentMap(); private final List<RemoteRepository> interpreterRepositories; private InterpreterOption defaultOption; @@ -117,20 +110,23 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean private ConfigStorage configStorage; private RemoteInterpreterEventServer interpreterEventServer; - public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration, - AngularObjectRegistryListener angularObjectRegistryListener, - RemoteInterpreterProcessListener - remoteInterpreterProcessListener, - ApplicationEventListener appEventListener) + public InterpreterSettingManager( + ZeppelinConfiguration zeppelinConfiguration, + AngularObjectRegistryListener angularObjectRegistryListener, + RemoteInterpreterProcessListener remoteInterpreterProcessListener, + ApplicationEventListener appEventListener) throws IOException { - this(zeppelinConfiguration, new InterpreterOption(), + this( + zeppelinConfiguration, + new InterpreterOption(), angularObjectRegistryListener, remoteInterpreterProcessListener, appEventListener, ConfigStorage.getInstance(zeppelinConfiguration)); } - public InterpreterSettingManager(ZeppelinConfiguration conf, + public InterpreterSettingManager( + ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, @@ -178,7 +174,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean loadInterpreterSettingFromDefaultDir(false); Set<String> newlyAddedInterpreters = Sets.newHashSet(interpreterSettingTemplates.keySet()); newlyAddedInterpreters.removeAll(installedInterpreters); - if(!newlyAddedInterpreters.isEmpty()) { + if (!newlyAddedInterpreters.isEmpty()) { saveToFile(); } } catch (IOException e) { @@ -186,9 +182,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } } - private void initInterpreterSetting(InterpreterSetting interpreterSetting) { - interpreterSetting.setConf(conf) + interpreterSetting + .setConf(conf) .setInterpreterSettingManager(this) .setAngularObjectRegistryListener(angularObjectRegistryListener) .setRemoteInterpreterProcessListener(remoteInterpreterProcessListener) @@ -200,12 +196,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean .postProcessing(); } - /** - * Load interpreter setting from interpreter.json - */ + /** Load interpreter setting from interpreter.json */ private void loadFromFile() throws IOException { - InterpreterInfoSaving infoSaving = - configStorage.loadInterpreterSettings(); + InterpreterInfoSaving infoSaving = configStorage.loadInterpreterSettings(); if (infoSaving == null) { // it is fresh zeppelin instance if there's no interpreter.json, just create interpreter // setting from interpreterSettingTemplates @@ -217,11 +210,10 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return; } - //TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving) + // TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving) for (InterpreterSetting savedInterpreterSetting : infoSaving.interpreterSettings.values()) { - savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties( - savedInterpreterSetting.getProperties() - )); + savedInterpreterSetting.setProperties( + InterpreterSetting.convertInterpreterProperties(savedInterpreterSetting.getProperties())); initInterpreterSetting(savedInterpreterSetting); InterpreterSetting interpreterSettingTemplate = @@ -233,14 +225,16 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean savedInterpreterSetting.setInterpreterDir(interpreterSettingTemplate.getInterpreterDir()); // merge properties from interpreter-setting.json and interpreter.json Map<String, InterpreterProperty> mergedProperties = - new HashMap<>(InterpreterSetting.convertInterpreterProperties( - interpreterSettingTemplate.getProperties())); - Map<String, InterpreterProperty> savedProperties = InterpreterSetting - .convertInterpreterProperties(savedInterpreterSetting.getProperties()); + new HashMap<>( + InterpreterSetting.convertInterpreterProperties( + interpreterSettingTemplate.getProperties())); + Map<String, InterpreterProperty> savedProperties = + InterpreterSetting.convertInterpreterProperties( + savedInterpreterSetting.getProperties()); for (Map.Entry<String, InterpreterProperty> entry : savedProperties.entrySet()) { // only merge properties whose value is not empty - if (entry.getValue().getValue() != null && ! - StringUtils.isBlank(entry.getValue().toString())) { + if (entry.getValue().getValue() != null + && !StringUtils.isBlank(entry.getValue().toString())) { mergedProperties.put(entry.getKey(), entry.getValue()); } } @@ -251,9 +245,11 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean savedInterpreterSetting.setInterpreterRunner( interpreterSettingTemplate.getInterpreterRunner()); } else { - LOGGER.warn("No InterpreterSetting Template found for InterpreterSetting: " - + savedInterpreterSetting.getGroup() + ", but it is found in interpreter.json, " - + "it would be skipped."); + LOGGER.warn( + "No InterpreterSetting Template found for InterpreterSetting: " + + savedInterpreterSetting.getGroup() + + ", but it is found in interpreter.json, " + + "it would be skipped."); continue; } @@ -265,8 +261,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } } savedInterpreterSetting.postProcessing(); - LOGGER.info("Create Interpreter Setting {} from interpreter.json", - savedInterpreterSetting.getName()); + LOGGER.info( + "Create Interpreter Setting {} from interpreter.json", savedInterpreterSetting.getName()); interpreterSettings.put(savedInterpreterSetting.getId(), savedInterpreterSetting); } @@ -305,24 +301,24 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean String interpreterJson = conf.getInterpreterJson(); ClassLoader cl = Thread.currentThread().getContextClassLoader(); if (Files.exists(interpreterDirPath)) { - for (Path interpreterDir : Files - .newDirectoryStream(interpreterDirPath, new Filter<Path>() { - @Override - public boolean accept(Path entry) throws IOException { - return Files.exists(entry) && Files.isDirectory(entry); - } - })) { + for (Path interpreterDir : + Files.newDirectoryStream( + interpreterDirPath, + new Filter<Path>() { + @Override + public boolean accept(Path entry) throws IOException { + return Files.exists(entry) && Files.isDirectory(entry); + } + })) { String interpreterDirString = interpreterDir.toString(); /** - * Register interpreter by the following ordering - * 1. Register it from path {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ - * interpreter-setting.json - * 2. Register it from interpreter-setting.json in classpath - * {ZEPPELIN_HOME}/interpreter/{interpreter_name} + * Register interpreter by the following ordering 1. Register it from path + * {ZEPPELIN_HOME}/interpreter/{interpreter_name}/ interpreter-setting.json 2. Register it + * from interpreter-setting.json in classpath {ZEPPELIN_HOME}/interpreter/{interpreter_name} */ if (!registerInterpreterFromPath(interpreterDirString, interpreterJson, override)) { - if (!registerInterpreterFromResource(cl, interpreterDirString, interpreterJson, - override)) { + if (!registerInterpreterFromResource( + cl, interpreterDirString, interpreterJson, override)) { LOGGER.warn("No interpreter-setting.json found in " + interpreterDirString); } } @@ -340,8 +336,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return appEventListener; } - private boolean registerInterpreterFromResource(ClassLoader cl, String interpreterDir, - String interpreterJson, boolean override) throws IOException { + private boolean registerInterpreterFromResource( + ClassLoader cl, String interpreterDir, String interpreterJson, boolean override) + throws IOException { URL[] urls = recursiveBuildLibList(new File(interpreterDir)); ClassLoader tempClassLoader = new URLClassLoader(urls, null); @@ -357,8 +354,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return true; } - private boolean registerInterpreterFromPath(String interpreterDir, String interpreterJson, - boolean override) throws IOException { + private boolean registerInterpreterFromPath( + String interpreterDir, String interpreterJson, boolean override) throws IOException { Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); if (Files.exists(interpreterJsonPath)) { @@ -372,13 +369,12 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } private List<RegisteredInterpreter> getInterpreterListFromJson(InputStream stream) { - Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() { - }.getType(); + Type registeredInterpreterListType = new TypeToken<List<RegisteredInterpreter>>() {}.getType(); return gson.fromJson(new InputStreamReader(stream), registeredInterpreterListType); } - private void registerInterpreterSetting(List<RegisteredInterpreter> registeredInterpreters, - String interpreterDir, boolean override) { + private void registerInterpreterSetting( + List<RegisteredInterpreter> registeredInterpreters, String interpreterDir, boolean override) { Map<String, DefaultInterpreterProperty> properties = new HashMap<>(); List<InterpreterInfo> interpreterInfos = new ArrayList<>(); @@ -386,10 +382,13 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean String group = null; InterpreterRunner runner = null; for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { - //TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo + // TODO(zjffdu) merge RegisteredInterpreter & InterpreterInfo InterpreterInfo interpreterInfo = - new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), - registeredInterpreter.isDefaultInterpreter(), registeredInterpreter.getEditor()); + new InterpreterInfo( + registeredInterpreter.getClassName(), + registeredInterpreter.getName(), + registeredInterpreter.isDefaultInterpreter(), + registeredInterpreter.getEditor()); group = registeredInterpreter.getGroup(); runner = registeredInterpreter.getRunner(); // use defaultOption if it is not specified in interpreter-setting.json @@ -400,22 +399,23 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean interpreterInfos.add(interpreterInfo); } - InterpreterSetting interpreterSettingTemplate = new InterpreterSetting.Builder() - .setGroup(group) - .setName(group) - .setInterpreterInfos(interpreterInfos) - .setProperties(properties) - .setDependencies(new ArrayList<Dependency>()) - .setOption(option) - .setRunner(runner) - .setInterpreterDir(interpreterDir) - .setRunner(runner) - .setConf(conf) - .setIntepreterSettingManager(this) - .create(); + InterpreterSetting interpreterSettingTemplate = + new InterpreterSetting.Builder() + .setGroup(group) + .setName(group) + .setInterpreterInfos(interpreterInfos) + .setProperties(properties) + .setDependencies(new ArrayList<Dependency>()) + .setOption(option) + .setRunner(runner) + .setInterpreterDir(interpreterDir) + .setRunner(runner) + .setConf(conf) + .setIntepreterSettingManager(this) + .create(); String key = interpreterSettingTemplate.getName(); - if(override || !interpreterSettingTemplates.containsKey(key)) { + if (override || !interpreterSettingTemplates.containsKey(key)) { LOGGER.info("Register InterpreterSettingTemplate: {}", key); interpreterSettingTemplates.put(key, interpreterSettingTemplate); } @@ -453,9 +453,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return null; } - //TODO(zjffdu) logic here is a little ugly - public Map<String, Object> getEditorSetting(Interpreter interpreter, String user, String noteId, - String replName) { + // TODO(zjffdu) logic here is a little ugly + public Map<String, Object> getEditorSetting( + Interpreter interpreter, String user, String noteId, String replName) { Map<String, Object> editor = DEFAULT_EDITOR; String group = StringUtils.EMPTY; try { @@ -491,7 +491,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return interpreterGroups; } - //TODO(zjffdu) move Resource related api to ResourceManager + // TODO(zjffdu) move Resource related api to ResourceManager public ResourceSet getAllResources() { return getAllResourcesExcept(null); } @@ -499,8 +499,8 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean private ResourceSet getAllResourcesExcept(String interpreterGroupExcludsion) { ResourceSet resourceSet = new ResourceSet(); for (ManagedInterpreterGroup intpGroup : getAllInterpreterGroup()) { - if (interpreterGroupExcludsion != null && - intpGroup.getId().equals(interpreterGroupExcludsion)) { + if (interpreterGroupExcludsion != null + && intpGroup.getId().equals(interpreterGroupExcludsion)) { continue; } @@ -511,13 +511,15 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean resourceSet.addAll(localPool.getAll()); } } else if (remoteInterpreterProcess.isRunning()) { - List<String> resourceList = remoteInterpreterProcess.callRemoteFunction( - new RemoteInterpreterProcess.RemoteFunction<List<String>>() { - @Override - public List<String> call(RemoteInterpreterService.Client client) throws Exception { - return client.resourcePoolGetAll(); - } - }); + List<String> resourceList = + remoteInterpreterProcess.callRemoteFunction( + new RemoteInterpreterProcess.RemoteFunction<List<String>>() { + @Override + public List<String> call(RemoteInterpreterService.Client client) + throws Exception { + return client.resourcePoolGetAll(); + } + }); if (resourceList != null) { for (String res : resourceList) { resourceSet.add(Resource.fromJson(res)); @@ -555,13 +557,15 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean r.getResourceId().getName()); } } else if (remoteInterpreterProcess.isRunning()) { - List<String> resourceList = remoteInterpreterProcess.callRemoteFunction( - new RemoteInterpreterProcess.RemoteFunction<List<String>>() { - @Override - public List<String> call(RemoteInterpreterService.Client client) throws Exception { - return client.resourcePoolGetAll(); - } - }); + List<String> resourceList = + remoteInterpreterProcess.callRemoteFunction( + new RemoteInterpreterProcess.RemoteFunction<List<String>>() { + @Override + public List<String> call(RemoteInterpreterService.Client client) + throws Exception { + return client.resourcePoolGetAll(); + } + }); for (String res : resourceList) { resourceSet.add(Resource.fromJson(res)); } @@ -601,42 +605,45 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean */ private void copyDependenciesFromLocalPath(final InterpreterSetting setting) { setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES); - final Thread t = new Thread() { - public void run() { - try { - List<Dependency> deps = setting.getDependencies(); - if (deps != null) { - for (Dependency d : deps) { - File destDir = new File( - conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); - - int numSplits = d.getGroupArtifactVersion().split(":").length; - if (!(numSplits >= 3 && numSplits <= 6)) { - dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(), - new File(destDir, setting.getId())); + final Thread t = + new Thread() { + public void run() { + try { + List<Dependency> deps = setting.getDependencies(); + if (deps != null) { + for (Dependency d : deps) { + File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + + int numSplits = d.getGroupArtifactVersion().split(":").length; + if (!(numSplits >= 3 && numSplits <= 6)) { + dependencyResolver.copyLocalDependency( + d.getGroupArtifactVersion(), new File(destDir, setting.getId())); + } } } - } - setting.setStatus(InterpreterSetting.Status.READY); - } catch (Exception e) { - LOGGER.error(String.format("Error while copying deps for interpreter group : %s," + - " go to interpreter setting page click on edit and save it again to make " + - "this interpreter work properly.", - setting.getGroup()), e); - setting.setErrorReason(e.getLocalizedMessage()); - setting.setStatus(InterpreterSetting.Status.ERROR); - } finally { + setting.setStatus(InterpreterSetting.Status.READY); + } catch (Exception e) { + LOGGER.error( + String.format( + "Error while copying deps for interpreter group : %s," + + " go to interpreter setting page click on edit and save it again to make " + + "this interpreter work properly.", + setting.getGroup()), + e); + setting.setErrorReason(e.getLocalizedMessage()); + setting.setStatus(InterpreterSetting.Status.ERROR); + } finally { + } } - } - }; - t.start(); + }; + t.start(); } /** - * Return ordered interpreter setting list. - * The list does not contain more than one setting from the same interpreter class. - * Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), Interpreter setting name + * Return ordered interpreter setting list. The list does not contain more than one setting from + * the same interpreter class. Order by InterpreterClass (order defined by ZEPPELIN_INTERPRETERS), + * Interpreter setting name */ public List<String> getInterpreterSettingIds() { List<String> settingIdList = new ArrayList<>(); @@ -646,8 +653,12 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return settingIdList; } - public InterpreterSetting createNewSetting(String name, String group, - List<Dependency> dependencies, InterpreterOption option, Map<String, InterpreterProperty> p) + public InterpreterSetting createNewSetting( + String name, + String group, + List<Dependency> dependencies, + InterpreterOption option, + Map<String, InterpreterProperty> p) throws IOException { if (name.indexOf(".") >= 0) { @@ -662,7 +673,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean InterpreterSetting setting = new InterpreterSetting(interpreterSettingTemplates.get(group)); setting.setName(name); setting.setGroup(group); - //TODO(zjffdu) Should use setDependencies + // TODO(zjffdu) Should use setDependencies setting.appendDependencies(dependencies); setting.setInterpreterOption(option); setting.setProperties(p); @@ -672,8 +683,6 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return setting; } - - @VisibleForTesting public void closeNote(String user, String noteId) { // close interpreters in this note session @@ -703,7 +712,7 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean } return urls; } else { - return new URL[]{path.toURI().toURL()}; + return new URL[] {path.toURI().toURL()}; } } @@ -711,8 +720,9 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean return this.interpreterRepositories; } - public void addRepository(String id, String url, boolean snapshot, Authentication auth, - Proxy proxy) throws IOException { + public void addRepository( + String id, String url, boolean snapshot, Authentication auth, Proxy proxy) + throws IOException { dependencyResolver.addRepo(id, url, snapshot, auth, proxy); saveToFile(); } @@ -797,23 +807,23 @@ public class InterpreterSettingManager implements InterpreterSettingManagerMBean FileUtils.deleteDirectory(localRepoDir); } - /** - * Get interpreter settings - */ + /** Get interpreter settings */ public List<InterpreterSetting> get() { List<InterpreterSetting> orderedSettings = new ArrayList<>(interpreterSettings.values()); - Collections.sort(orderedSettings, new Comparator<InterpreterSetting>() { - @Override - public int compare(InterpreterSetting o1, InterpreterSetting o2) { - if (o1.getName().equals(defaultInterpreterGroup)) { - return -1; - } else if (o2.getName().equals(defaultInterpreterGroup)) { - return 1; - } else { - return o1.getName().compareTo(o2.getName()); - } - } - }); + Collections.sort( + orderedSettings, + new Comparator<InterpreterSetting>() { + @Override + public int compare(InterpreterSetting o1, InterpreterSetting o2) { + if (o1.getName().equals(defaultInterpreterGroup)) { + return -1; + } else if (o2.getName().equals(defaultInterpreterGroup)) { + return 1; + } else { + return o1.getName().compareTo(o2.getName()); + } + } + }); return orderedSettings; } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java index f36cb0d..adcdcd4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/LifecycleManager.java @@ -15,18 +15,12 @@ * limitations under the License. */ - package org.apache.zeppelin.interpreter; - -/** - * Interface for managing the lifecycle of interpreters - */ +/** Interface for managing the lifecycle of interpreters */ public interface LifecycleManager { void onInterpreterProcessStarted(ManagedInterpreterGroup interpreterGroup); - void onInterpreterUse(ManagedInterpreterGroup interpreterGroup, - String sessionId); - + void onInterpreterUse(ManagedInterpreterGroup interpreterGroup, String sessionId); } http://git-wip-us.apache.org/repos/asf/zeppelin/blob/55f6c91c/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java ---------------------------------------------------------------------- diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java index e1470df..c6d5216 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java @@ -15,10 +15,12 @@ * limitations under the License. */ - package org.apache.zeppelin.interpreter; -import org.apache.commons.lang.exception.ExceptionUtils; +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Properties; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Scheduler; @@ -26,14 +28,7 @@ import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Properties; - -/** - * ManagedInterpreterGroup runs under zeppelin server - */ +/** ManagedInterpreterGroup runs under zeppelin server */ public class ManagedInterpreterGroup extends InterpreterGroup { private static final Logger LOGGER = LoggerFactory.getLogger(ManagedInterpreterGroup.class); @@ -43,6 +38,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup { /** * Create InterpreterGroup with given id and interpreterSetting, used in ZeppelinServer + * * @param id * @param interpreterSetting */ @@ -55,16 +51,16 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return interpreterSetting; } - public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess(String userName, - Properties properties) - throws IOException { + public synchronized RemoteInterpreterProcess getOrCreateInterpreterProcess( + String userName, Properties properties) throws IOException { if (remoteInterpreterProcess == null) { LOGGER.info("Create InterpreterProcess for InterpreterGroup: " + getId()); - remoteInterpreterProcess = interpreterSetting.createInterpreterProcess(id, userName, - properties); + remoteInterpreterProcess = + interpreterSetting.createInterpreterProcess(id, userName, properties); remoteInterpreterProcess.start(userName); interpreterSetting.getLifecycleManager().onInterpreterProcessStarted(this); - getInterpreterSetting().getRecoveryStorage() + getInterpreterSetting() + .getRecoveryStorage() .onInterpreterClientStart(remoteInterpreterProcess); } return remoteInterpreterProcess; @@ -78,10 +74,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return remoteInterpreterProcess; } - - /** - * Close all interpreter instances in this group - */ + /** Close all interpreter instances in this group */ public synchronized void close() { LOGGER.info("Close InterpreterGroup: " + id); for (String sessionId : sessions.keySet()) { @@ -91,13 +84,17 @@ public class ManagedInterpreterGroup extends InterpreterGroup { /** * Close all interpreter instances in this session + * * @param sessionId */ public synchronized void close(String sessionId) { - LOGGER.info("Close Session: " + sessionId + " for interpreter setting: " + - interpreterSetting.getName()); + LOGGER.info( + "Close Session: " + + sessionId + + " for interpreter setting: " + + interpreterSetting.getName()); close(sessions.remove(sessionId)); - //TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server + // TODO(zjffdu) whether close InterpreterGroup if there's no session left in Zeppelin Server if (sessions.isEmpty() && interpreterSetting != null) { LOGGER.info("Remove this InterpreterGroup: {} as all the sessions are closed", id); interpreterSetting.removeInterpreterGroup(id); @@ -137,7 +134,7 @@ public class ManagedInterpreterGroup extends InterpreterGroup { } catch (InterpreterException e) { LOGGER.warn("Fail to close interpreter " + interpreter.getClassName(), e); } - //TODO(zjffdu) move the close of schedule to Interpreter + // TODO(zjffdu) move the close of schedule to Interpreter if (null != scheduler) { SchedulerFactory.singleton().removeScheduler(scheduler.getName()); } @@ -157,5 +154,4 @@ public class ManagedInterpreterGroup extends InterpreterGroup { return interpreters; } } - }
