Changed occurrences of Myriad and Hadoop versions Myriad: 0.0.1 -> 0.1.0 Hadoop: 2.7.1 -> 2.7.0
This closes: #37 Review: https://github.com/apache/incubator-myriad/pull/37 Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/4204ee24 Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/4204ee24 Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/4204ee24 Branch: refs/heads/master Commit: 4204ee24bf5a7c0a7bd4e865b74865d49b3d3278 Parents: dbd8860 Author: smarella <smare...@maprtech.com> Authored: Mon Nov 9 20:07:22 2015 -0800 Committer: smarella <smare...@maprtech.com> Committed: Mon Nov 9 20:07:22 2015 -0800 ---------------------------------------------------------------------- build.gradle | 2 +- docker/Dockerfile | 4 ++-- docker/ResourceManager.dockerfile | 4 ++-- docs/API.md | 4 ++-- docs/getting-started.md | 2 +- docs/install-overview.md | 2 +- docs/myriad-configuration.md | 2 +- docs/myriad-dev.md | 6 +++--- .../myriad-remote-distribution-configuration.md | 22 ++++++++++---------- docs/vagrant.md | 6 +++--- myriad-scheduler/build.gradle | 8 +++---- .../configuration/MyriadConfiguration.java | 2 +- .../main/resources/myriad-config-default.yml | 8 +++---- .../scheduler/TestServiceCommandLine.java | 6 +++--- .../resources/myriad-config-test-default.yml | 10 ++++----- setup-yarn-1.sh | 4 ++-- 16 files changed, 46 insertions(+), 46 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 4b62404..807d117 100644 --- a/build.gradle +++ b/build.gradle @@ -49,7 +49,7 @@ subprojects { mainClassName = "org.apache.myriad.Main" group = "org.apache.myriad" - version = "0.0.1" + version = "0.1.0" ext { mesosVer = "0.24.1" http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/Dockerfile b/docker/Dockerfile index 9a89838..d927e81 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # Myriad - YARN Resource Manager # -# VERSION 0.0.1 +# VERSION 0.1.0 FROM debian MAINTAINER Apache Myriad d...@myriad.incubator.apache.org @@ -26,7 +26,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh RUN sh /install-yarn.sh # Copy over myriad libraries -ADD /libs/myriad-executor-runnable-0.0.1.jar /usr/local/libexec/mesos/ +ADD /libs/myriad-executor-runnable-0.1.0.jar /usr/local/libexec/mesos/ ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docker/ResourceManager.dockerfile ---------------------------------------------------------------------- diff --git a/docker/ResourceManager.dockerfile b/docker/ResourceManager.dockerfile index f608ad9..4744c43 100644 --- a/docker/ResourceManager.dockerfile +++ b/docker/ResourceManager.dockerfile @@ -1,6 +1,6 @@ # Myriad - YARN Resource Manager # -# VERSION 0.0.1 +# VERSION 0.1.0 FROM debian MAINTAINER Apache Myriad d...@myriad.incubator.apache.org @@ -27,7 +27,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh RUN sh /install-yarn.sh # Copy over myriad libraries -ADD /libs/myriad-executor-runnable-0.0.1.jar /usr/local/libexec/mesos/ +ADD /libs/myriad-executor-runnable-0.1.0.jar /usr/local/libexec/mesos/ ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ # Initialize hadoop confs with env vars http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/API.md ---------------------------------------------------------------------- diff --git a/docs/API.md b/docs/API.md index e7d1555..f732210 100644 --- a/docs/API.md +++ b/docs/API.md @@ -416,7 +416,7 @@ Accept-Language: en-US,en;q=0.8 "haEnabled": false "restApiPort": 8192, "yarnEnvironment": { - "YARN_HOME": "/root/hadoop-2.7.1" + "YARN_HOME": "/root/hadoop-2.7.0" }, "zkServers": "10.10.100.19:2181", "zkTimeout": 20000 @@ -481,4 +481,4 @@ Accept-Language: en-US,en;q=0.8 ], "stagingTasks": [] } -``` \ No newline at end of file +``` http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/getting-started.md ---------------------------------------------------------------------- diff --git a/docs/getting-started.md b/docs/getting-started.md index b983a98..064d814 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -45,7 +45,7 @@ yarn-daemon.sh stop resourcemanager Alternatively, start and stop Myriad from the Marathon UI. See Marathon: Application Basics for more information. For example, create an application to start the Resource Manager: ``` -cd hadoop-2.7.1/sbin && yarn-daemon.sh start resourcemanager +cd hadoop-2.7.0/sbin && yarn-daemon.sh start resourcemanager ``` Alternatively, when launching the Resource Manager in an HA environment, specify value for the `yarn.resourcemanager.hostname` property. The hostname is the ID field specified when launching a Marathon application. http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/install-overview.md ---------------------------------------------------------------------- diff --git a/docs/install-overview.md b/docs/install-overview.md index 6d7479b..32fae6f 100644 --- a/docs/install-overview.md +++ b/docs/install-overview.md @@ -4,7 +4,7 @@ * JDK 1.7+ * Gradle -* Hadoop 2.7.1 +* Hadoop 2.7.0 ## Operating Systems ## http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/myriad-configuration.md ---------------------------------------------------------------------- diff --git a/docs/myriad-configuration.md b/docs/myriad-configuration.md index a1f9263..5bdfb4a 100644 --- a/docs/myriad-configuration.md +++ b/docs/myriad-configuration.md @@ -60,7 +60,7 @@ executor: # nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz # the uri to d/l hadoop from # Path to the Hadoop tarball # Environment variables required to launch Node Manager process. Admin can also pass other environment variables to NodeManager. yarnEnvironment: - YARN_HOME: /usr/local/hadoop # Or /opt/mapr/hadoop/hadoop-2.7.1/ if using MapR's Hadoop + YARN_HOME: /usr/local/hadoop # Or /opt/mapr/hadoop/hadoop-2.7.0/ if using MapR's Hadoop # YARN_HOME: hadoop-2.7.0 # Should be relative nodeManagerUri is set YARN_NODEMANAGER_OPTS: -Dnodemanager.resource.io-spindles=4.0 # Required only if using MapR's Hadoop # JAVA_HOME: /usr/lib/jvm/java-default # System dependent, but sometimes necessary http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/myriad-dev.md ---------------------------------------------------------------------- diff --git a/docs/myriad-dev.md b/docs/myriad-dev.md index d8eb000..33c0d8b 100644 --- a/docs/myriad-dev.md +++ b/docs/myriad-dev.md @@ -63,9 +63,9 @@ To deploy Myriad Scheduler and Executor files: For example: ``` -cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.7.1/share/hadoop/yarn/lib/ -cp myriad-executor/build/libs/myriad-executor-0.0.1.jar /opt/hadoop-2.7.1/share/hadoop/yarn/lib/ -cp myriad-scheduler/build/resources/main/myriad-config-default.yml /opt/hadoop-2.7.1/etc/hadoop/ +cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.7.0/share/hadoop/yarn/lib/ +cp myriad-executor/build/libs/myriad-executor-0.1.0.jar /opt/hadoop-2.7.0/share/hadoop/yarn/lib/ +cp myriad-scheduler/build/resources/main/myriad-config-default.yml /opt/hadoop-2.7.0/etc/hadoop/ ``` **NOTE:** For advanced users, you can also copy myriad-executor-xxx.jar to any other directory on a slave filesystem or it can be copied to HDFS as well. In either case, you need to update the executor's path property in the myriad-config-default.yml file and prepend the path with either file:// or hdfs://, as appropriate. http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/myriad-remote-distribution-configuration.md ---------------------------------------------------------------------- diff --git a/docs/myriad-remote-distribution-configuration.md b/docs/myriad-remote-distribution-configuration.md index 632568a..a3657fe 100644 --- a/docs/myriad-remote-distribution-configuration.md +++ b/docs/myriad-remote-distribution-configuration.md @@ -16,9 +16,9 @@ This information involves bundling Myriad and creating a tarball. The following are assumptions about your environment: -* You are using hadoop-2.7.1 downloaded from [hadoop.apache.org](http://hadoop.apache.org). Specific vendor versions should work but may require additional steps. +* You are using hadoop-2.7.0 downloaded from [hadoop.apache.org](http://hadoop.apache.org). Specific vendor versions should work but may require additional steps. -**NOTE:** The default location for $YARN_HOME is **/opt/hadoop-2.7.1**. +**NOTE:** The default location for $YARN_HOME is **/opt/hadoop-2.7.0**. ## Building the Myriad Remote Distribution Bundle ## Before building Myriad, configure the Resource Manager as you normally would. @@ -35,9 +35,9 @@ From the project root you build Myriad with the commands To deploy the Myriad Scheduler and Executor files, copy the jar filess and configuration file to the following locations: ``` -cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.7.1/share/hadoop/yarn/lib/ -cp myriad-executor/build/libs/myriad-executor-0.0.1.jar /opt/hadoop-2.7.1/share/hadoop/yarn/lib/ -cp myriad-scheduler/build/resources/main/myriad-config-default.yml /opt/hadoop-2.7.1/etc/hadoop/ +cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.7.0/share/hadoop/yarn/lib/ +cp myriad-executor/build/libs/myriad-executor-0.1.0.jar /opt/hadoop-2.7.0/share/hadoop/yarn/lib/ +cp myriad-scheduler/build/resources/main/myriad-config-default.yml /opt/hadoop-2.7.0/etc/hadoop/ ``` ### Step 3: Configure the Myriad Defaults @@ -50,9 +50,9 @@ frameworkSuperUser: admin # Must be root or have passwordless sudo frameworkUser: hduser # Should be the same user running the resource manager. # Must exist on all nodes and be in the 'hadoop' group executor: - nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.1.tar.gz + nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz yarnEnvironment: -YARN_HOME: hadoop-2.7.1 # This should be relative if nodeManagerUri is set +YARN_HOME: hadoop-2.7.0 # This should be relative if nodeManagerUri is set ``` @@ -68,10 +68,10 @@ The tarball has all of the files needed for the Node Managers and Resource Mana ``` cd ~ -sudo cp -rp /opt/hadoop-2.7.1 . -sudo rm ~/hadoop-2.7.1/etc/hadoop/yarn-site.xml -sudo tar -zcpf ~/hadoop-2.7.1.tar.gz hadoop-2.7.1 -hadoop fs -put ~/hadoop-2.7.1.tar.gz /dist +sudo cp -rp /opt/hadoop-2.7.0 . +sudo rm ~/hadoop-2.7.0/etc/hadoop/yarn-site.xml +sudo tar -zcpf ~/hadoop-2.7.0.tar.gz hadoop-2.7.0 +hadoop fs -put ~/hadoop-2.7.0.tar.gz /dist ``` ## Getting Started ## http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/docs/vagrant.md ---------------------------------------------------------------------- diff --git a/docs/vagrant.md b/docs/vagrant.md index c42447f..b30731d 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -97,14 +97,14 @@ The Myriad Schedule and Executer jar files and all the runtime dependences as we * The Myriad Executor jar file are located at: ``` -/vagrant/myriad-scheduler/build/libs/myriad-executor-0.0.1.jar +/vagrant/myriad-scheduler/build/libs/myriad-executor-0.1.0.jar ``` For example, the files are copied to the following locations: ``` cp /vagrant/myriad-scheduler/build/libs/* $YARN_HOME/share/hadoop/yarn/lib/ -cp /vagrant/myriad-executor/build/libs/myriad-executor-0.0.1.jar $YARN_HOME/share/hadoop/yarn/lib/ +cp /vagrant/myriad-executor/build/libs/myriad-executor-0.1.0.jar $YARN_HOME/share/hadoop/yarn/lib/ cp /vagrant/myriad-scheduler/src/main/resources/myriad-config-default.yml $YARN_HOME/etc/hadoop/ ``` @@ -147,7 +147,7 @@ nodemanager: cgroups: false executor: jvmMaxMemoryMB: 256 - path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar + path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar ``` http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/myriad-scheduler/build.gradle ---------------------------------------------------------------------- diff --git a/myriad-scheduler/build.gradle b/myriad-scheduler/build.gradle index 63d69ed..fc7516d 100644 --- a/myriad-scheduler/build.gradle +++ b/myriad-scheduler/build.gradle @@ -104,9 +104,9 @@ gulp { // Whether output from Gulp should be buffered - useful when running tasks in parallel bufferOutput = false - // processes your package.json before running gulp build - gulp_build.dependsOn 'npmInstall' - - // makes sure on each build that gulp is installed + // makes sure on each build that gulp is installed gulp_build.dependsOn 'installGulp' + + // processes your package.json before running gulp build + gulp_build.dependsOn 'npmInstall' } http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java index 78df3b6..67c8232 100644 --- a/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java +++ b/myriad-scheduler/src/main/java/org/apache/myriad/configuration/MyriadConfiguration.java @@ -52,7 +52,7 @@ import org.hibernate.validator.constraints.NotEmpty; * cgroups: false * executor: * jvmMaxMemoryMB: 256 - * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar + * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar * yarnEnvironment: * YARN_HOME: /usr/local/hadoop */ http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/myriad-scheduler/src/main/resources/myriad-config-default.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml index 4f761af..ed4bef9 100644 --- a/myriad-scheduler/src/main/resources/myriad-config-default.yml +++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml @@ -52,13 +52,13 @@ nodemanager: cgroups: false executor: jvmMaxMemoryMB: 256 - path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar + path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. - #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz - #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar + #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz + #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar yarnEnvironment: YARN_HOME: /usr/local/hadoop - #YARN_HOME: hadoop-2.5.0 #this should be relative if nodeManagerUri is set + #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary mesosAuthenticationPrincipal: mesosAuthenticationSecretFilename: http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java index 9d697fe..ab7e7d9 100644 --- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java +++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/TestServiceCommandLine.java @@ -37,12 +37,12 @@ public class TestServiceCommandLine { static MyriadConfiguration cfg; static String toJHSCompare = - "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + + "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; " + "sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver\";sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp" + " conf /usr/local/hadoop/etc/hadoop/yarn-site.xml; sudo -E -u hduser -H $YARN_HOME/bin/mapred historyserver"; static String toCompare = - "echo \"sudo tar -zxpf hadoop-2.5.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; + "echo \"sudo tar -zxpf hadoop-2.7.0.tar.gz && sudo chown hduser . && cp conf /usr/local/hadoop/etc/hadoop/yarn-site.xml;"; @BeforeClass public static void setUpBeforeClass() throws Exception { @@ -75,7 +75,7 @@ public class TestServiceCommandLine { ServiceResourceProfile profile = new ExtendedResourceProfile(new NMProfile("nm", 10L, 15L), 3.0, 5.0); ExecutorCommandLineGenerator clGenerator = new DownloadNMExecutorCLGenImpl(cfg, - "hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz"); + "hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz"); NMTaskFactoryImpl nms = new NMTaskFactoryImpl(cfg, null, clGenerator); CommandInfo cInfo = nms.getCommandInfo(profile, nmPorts); http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/myriad-scheduler/src/test/resources/myriad-config-test-default.yml ---------------------------------------------------------------------- diff --git a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml index 50003d7..ab597f6 100644 --- a/myriad-scheduler/src/test/resources/myriad-config-test-default.yml +++ b/myriad-scheduler/src/test/resources/myriad-config-test-default.yml @@ -28,13 +28,13 @@ nodemanager: cgroups: false executor: jvmMaxMemoryMB: 256 - path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar + path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.1.0.jar #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. - nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz - #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar + nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.7.0.tar.gz + #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.1.0.jar yarnEnvironment: YARN_HOME: /usr/local/hadoop - #YARN_HOME: hadoop-2.5.0 #this should be relative if nodeManagerUri is set + #YARN_HOME: hadoop-2.7.0 #this should be relative if nodeManagerUri is set #JAVA_HOME: /usr/lib/jvm/java-default #System dependent, but sometimes necessary mesosAuthenticationPrincipal: mesosAuthenticationSecretFilename: @@ -52,4 +52,4 @@ services: jvmMaxMemoryMB: 1024 cpus: 1 envSettings: -Dcluster.name.prefix=/mycluster2 - taskName: timelineserver \ No newline at end of file + taskName: timelineserver http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/4204ee24/setup-yarn-1.sh ---------------------------------------------------------------------- diff --git a/setup-yarn-1.sh b/setup-yarn-1.sh index 831ba4e..314d5cc 100755 --- a/setup-yarn-1.sh +++ b/setup-yarn-1.sh @@ -1,6 +1,6 @@ set -e -HADOOP_VER=2.5.0 +HADOOP_VER=2.7.0 cd ~/ @@ -15,4 +15,4 @@ sudo adduser hduser sudo # Setup password-less auth sudo -u hduser ssh-keygen -t rsa -P '' -sudo sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys' \ No newline at end of file +sudo sh -c 'cat /home/hduser/.ssh/id_rsa.pub >> /home/hduser/.ssh/authorized_keys'
