Repository: incubator-myriad Updated Branches: refs/heads/master bcd1fc8c6 -> 5118cff3e
Added the ability to specify a URI to pull configuration data from, also added to REST endpoints: /api/config.tgz and /api/binary.tgz which provide an easy download for the hadoop configs and the hadoop binarys. Configuration is done by assigning: servedConfigPath: dist/config.tgz servedBinaryPath: dist/hadoop-2.7.0.tgz and executor: configUri: http://127.0.0.1:8192/api/artifacts/config.tgz nodemanagerUri: http://127.0.0.1:8192/api/artifacts/binary.tgz Able to use, yarnEnvironment: HADOOP_CONF_DIR=$MESOS_SANDBOX/config YARN_HOME=$MESOS_SANDBOX/hadoop-2.7.0 JIRA: [Myriad-189] https://issues.apache.org/jira/browse/MYRIAD-189 Pull Request: Closes #68 Author: DarinJ <dar...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/5118cff3 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/5118cff3 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/5118cff3 Branch: refs/heads/master Commit: 5118cff3e0b3ba2e66023cd9a2840e74c546a6db Parents: bcd1fc8 Author: darinj <darinj.w...@gmail.com> Authored: Mon May 2 23:09:49 2016 -0400 Committer: darinj <dar...@apache.org> Committed: Wed May 11 21:56:45 2016 -0400 ---------------------------------------------------------------------- .../apache/myriad/api/ArtifactsResource.java | 78 ++++++++++++++++++++ .../configuration/MyriadConfiguration.java | 13 ++++ .../MyriadExecutorConfiguration.java | 8 ++ .../apache/myriad/scheduler/TaskFactory.java | 11 ++- .../myriad/webapp/MyriadServletModule.java | 3 +- .../main/resources/myriad-config-default.yml | 4 + 6 files changed, 113 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java b/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java new file mode 100644 index 0000000..769abe0 --- /dev/null +++ b/myriad-scheduler/src/main/java/org/apache/myriad/api/ArtifactsResource.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.myriad.api; + +import com.codahale.metrics.annotation.Timed; +import org.apache.myriad.configuration.MyriadConfiguration; + +import javax.inject.Inject; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.File; + +/** + * Defines the REST API to the Myriad configuration. + */ +@Path("/artifacts") +@Produces(MediaType.APPLICATION_OCTET_STREAM) +public class ArtifactsResource { + private String myriadEtc; + private String myriadBinary; + + @Inject + public ArtifactsResource(MyriadConfiguration cfg) { + myriadEtc = cfg.getServedConfigPath().or(""); + myriadBinary = cfg.getServedBinaryPath().or(""); + } + + @Timed + @GET + @Path("/config.tgz") + public Response getConfig() throws InterruptedException { + File file = new File(myriadEtc); + Response.ResponseBuilder response; + if (file.exists()) { + response = Response.ok((Object) file); + response.header("Content-Disposition", "attachment; filename=" + file.getName()); + } else { + response = Response.status(Response.Status.BAD_REQUEST) + .entity("Path does not exist"); + } + return response.build(); + } + + @Timed + @GET + @Path("/binary.tgz") + public Response getBinary() throws InterruptedException { + File file = new File(myriadBinary); + Response.ResponseBuilder response; + if (file.exists()) { + response = Response.ok((Object) file); + response.header("Content-Disposition", "attachment; filename=" + file.getName()); + } else { + response = Response.status(Response.Status.BAD_REQUEST) + .entity("Path does not exist"); + } + return response.build(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java index 716a20a..f65bb9a 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java @@ -179,6 +179,11 @@ public class MyriadConfiguration { @JsonProperty private String mesosAuthenticationSecretFilename; + @JsonProperty + private String servedConfigPath; + + @JsonProperty + private String servedBinaryPath; public MyriadConfiguration() { } @@ -278,4 +283,12 @@ public class MyriadConfiguration { return mesosAuthenticationPrincipal; } + public Optional<String> getServedConfigPath() { + return Optional.fromNullable(servedConfigPath); + } + + public Optional<String> getServedBinaryPath() { + return Optional.fromNullable(servedBinaryPath); + } + } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java index 6afc086..ea98ef7 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadExecutorConfiguration.java @@ -44,6 +44,10 @@ public class MyriadExecutorConfiguration { @JsonSerialize(using = OptionalSerializerString.class) private String nodeManagerUri; + @JsonProperty + @JsonSerialize(using = OptionalSerializerString.class) + private String configUri; + /** * Download URL for JRE. * Ex: jvmUri: http://www.apache.org/myriad/jre-1.8.99.tar.gz @@ -66,6 +70,10 @@ public class MyriadExecutorConfiguration { return Optional.fromNullable(nodeManagerUri); } + public Optional<String> getConfigUri() { + return Optional.fromNullable(configUri); + } + public Optional<String> getJvmUri() { return Optional.fromNullable(jvmUri); } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java index 9ed54a7..92e4cf7 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/scheduler/TaskFactory.java @@ -133,6 +133,12 @@ public interface TaskFactory { commandInfo.addUris(jvmUri); } + if (myriadExecutorConfiguration.getConfigUri().isPresent()) { + String configURI = myriadExecutorConfiguration.getConfigUri().get(); + LOGGER.info("Getting Hadoop distribution from: {}", configURI); + commandInfo.addUris(URI.newBuilder().setValue(configURI).build()); + } + if (myriadExecutorConfiguration.getNodeManagerUri().isPresent()) { //Both FrameworkUser and FrameworkSuperuser to get all of the directory permissions correct. if (!(cfg.getFrameworkUser().isPresent() && cfg.getFrameworkSuperUser().isPresent())) { @@ -143,14 +149,14 @@ public interface TaskFactory { //get the nodemanagerURI //We're going to extract ourselves, so setExtract is false - LOGGER.info("Getting Hadoop distribution from:" + nodeManagerUri); + LOGGER.info("Getting Hadoop distribution from: {}", nodeManagerUri); URI nmUri = URI.newBuilder().setValue(nodeManagerUri).setExtract(false).build(); //get configs directly from resource manager String configUrlString = clGenerator.getConfigurationUrl(); LOGGER.info("Getting config from:" + configUrlString); URI configUri = URI.newBuilder().setValue(configUrlString).build(); - LOGGER.info("Slave will execute command:" + cmd); + LOGGER.info("Slave will execute command: {}", cmd); commandInfo.addUris(nmUri).addUris(configUri).setValue("echo \"" + cmd + "\";" + cmd); commandInfo.setUser(cfg.getFrameworkSuperUser().get()); @@ -162,7 +168,6 @@ public interface TaskFactory { commandInfo.setUser(cfg.getFrameworkUser().get()); } } - return commandInfo.build(); } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java index 3aa3737..82d2bae 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/webapp/MyriadServletModule.java @@ -25,6 +25,7 @@ import org.apache.myriad.api.ClustersResource; import org.apache.myriad.api.ConfigurationResource; import org.apache.myriad.api.ControllerResource; import org.apache.myriad.api.SchedulerStateResource; +import org.apache.myriad.api.ArtifactsResource; import org.codehaus.jackson.jaxrs.JacksonJaxbJsonProvider; /** @@ -38,7 +39,7 @@ public class MyriadServletModule extends ServletModule { bind(ConfigurationResource.class); bind(SchedulerStateResource.class); bind(ControllerResource.class); - + bind(ArtifactsResource.class); bind(GuiceContainer.class); bind(JacksonJaxbJsonProvider.class).in(Scopes.SINGLETON); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/5118cff3/myriad-scheduler/src/main/resources/myriad-config-default.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml index a90de84..a2c143d 100644 --- a/myriad-scheduler/src/main/resources/myriad-config-default.yml +++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml @@ -29,6 +29,8 @@ nativeLibrary: /usr/local/lib/libmesos.so zkServers: localhost:2181 zkTimeout: 20000 restApiPort: 8192 +#servedConfigPath: dist/config.tgz +#servedBinaryPath: dist/hadoop-2.6.0.tgz profiles: zero: # NMs launched with this profile dynamically obtain cpu/mem from Mesos cpu: 0 @@ -55,10 +57,12 @@ executor: path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz + #configUri: http://127.0.0.1/api/arifacts/config.tgz #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar #jvmUri: https://downloads.mycompany.com/java/jre-7u76-linux-x64.tar.gz yarnEnvironment: YARN_HOME: /usr/local/hadoop + #HADOOP_CONF_DIR=config #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary #JAVA_HOME: jre1.7.0_76 # Path to JRE distribution, relative to sandbox directory
