Repository: incubator-myriad Updated Branches: refs/heads/master 1daad6324 -> f1bc1d5ee
Made JVM URI, Path and JAVA_LIBRARY_PATH configurable for executors JIRA: [Myriad-196] https://issues.apache.org/jira/browse/MYRIAD-196 Pull Request: Closes #65 Author: Mohit Soni <mohitsoni1...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/f1bc1d5e Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/f1bc1d5e Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/f1bc1d5e Branch: refs/heads/master Commit: f1bc1d5ee5cbf0a7e93c7922760fb4b2a973e519 Parents: 1daad63 Author: Mohit Soni <mohitsoni1...@gmail.com> Authored: Sun May 1 00:52:54 2016 -0700 Committer: darinj <dar...@apache.org> Committed: Mon May 9 14:46:19 2016 -0400 ---------------------------------------------------------------------- .../MyriadExecutorConfiguration.java | 14 ++++++++++ .../apache/myriad/scheduler/TaskFactory.java | 27 +++++++++----------- .../main/resources/myriad-config-default.yml | 3 +++ 3 files changed, 29 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java index 4479725..6afc086 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java @@ -44,6 +44,16 @@ public class MyriadExecutorConfiguration { @JsonSerialize(using = OptionalSerializerString.class) private String nodeManagerUri; + /** + * Download URL for JRE. + * Ex: jvmUri: http://www.apache.org/myriad/jre-1.8.99.tar.gz + * If provided, Mesos fetcher will be used to download and extract it + * inside sandbox. + */ + @JsonProperty + @JsonSerialize(using = OptionalSerializerString.class) + private String jvmUri; + public Optional<Double> getJvmMaxMemoryMB() { return Optional.fromNullable(jvmMaxMemoryMB); } @@ -55,4 +65,8 @@ public class MyriadExecutorConfiguration { public Optional<String> getNodeManagerUri() { return Optional.fromNullable(nodeManagerUri); } + + public Optional<String> getJvmUri() { + return Optional.fromNullable(jvmUri); + } } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java index 8412eba..ecf0276 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java @@ -20,22 +20,8 @@ package org.apache.myriad.scheduler; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Random; -import javax.inject.Inject; -import org.apache.mesos.Protos.CommandInfo; +import org.apache.mesos.Protos.*; import org.apache.mesos.Protos.CommandInfo.URI; -import org.apache.mesos.Protos.ExecutorID; -import org.apache.mesos.Protos.ExecutorInfo; -import org.apache.mesos.Protos.FrameworkID; -import org.apache.mesos.Protos.Offer; -import org.apache.mesos.Protos.Resource; -import org.apache.mesos.Protos.TaskID; -import org.apache.mesos.Protos.TaskInfo; -import org.apache.mesos.Protos.Value; import org.apache.mesos.Protos.Value.Range; import org.apache.myriad.configuration.MyriadConfiguration; import org.apache.myriad.configuration.MyriadExecutorConfiguration; @@ -43,6 +29,9 @@ import org.apache.myriad.state.NodeTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.inject.Inject; +import java.util.*; + /** * Creates Tasks based on mesos offers @@ -137,6 +126,13 @@ public interface TaskFactory { CommandInfo.Builder commandInfo = CommandInfo.newBuilder(); String cmd; + if (myriadExecutorConfiguration.getJvmUri().isPresent()) { + final String jvmRemoteUri = myriadExecutorConfiguration.getJvmUri().get(); + LOGGER.info("Getting JRE distribution from:" + jvmRemoteUri); + URI jvmUri = URI.newBuilder().setValue(jvmRemoteUri).setExtract(true).build(); + commandInfo.addUris(jvmUri); + } + if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) { //Both FrameworkUser and FrameworkSuperuser to get all of the directory permissions correct. if (!(cfg.getFrameworkUser().isPresent() && cfg.getFrameworkSuperUser().isPresent())) { @@ -166,6 +162,7 @@ public interface TaskFactory { commandInfo.setUser(cfg.getFrameworkUser().get()); } } + return commandInfo.build(); } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/f1bc1d5e/myriad-scheduler/src/main/resources/myriad-config-default.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml index ed4bef9..a90de84 100644 --- a/myriad-scheduler/src/main/resources/myriad-config-default.yml +++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml @@ -56,9 +56,12 @@ executor: #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar + #jvmUri: https://downloads.mycompany.com/java/jre-7u76-linux-x64.tar.gz yarnEnvironment: YARN_HOME: /usr/local/hadoop #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary + #JAVA_HOME: jre1.7.0_76 # Path to JRE distribution, relative to sandbox directory + #JAVA_LIBRARY_PATH: /opt/mycompany/lib mesosAuthenticationPrincipal: mesosAuthenticationSecretFilename:
