Repository: incubator-myriad Updated Branches: refs/heads/master 4bce035ec -> e1db4d8b8
Docker Gradle Integration Docker RM now is built via Gradle. Pull Request: Closes #55 Author: Brandon Gulla bran...@brandongulla.com Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/e1db4d8b Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/e1db4d8b Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/e1db4d8b Branch: refs/heads/master Commit: e1db4d8b8cdb4fd1994be9097ab6e439e8f28cc0 Parents: 4bce035 Author: Brandon Gulla <bran...@brandongulla.com> Authored: Wed Nov 18 12:51:49 2015 -0500 Committer: darinj <darinj.w...@gmail.com> Committed: Wed Mar 30 23:51:13 2016 -0400 ---------------------------------------------------------------------- build.gradle | 15 ++++++++++ docker/Dockerfile | 2 +- docker/README.md | 20 +++++-------- docker/ResourceManager.dockerfile | 55 ---------------------------------- docker/build.gradle | 50 +++++++++++++++++++++++++++++++ docker/myriad-bin/create-user.sh | 2 +- settings.gradle | 1 + 7 files changed, 75 insertions(+), 70 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/build.gradle ---------------------------------------------------------------------- diff --git a/build.gradle b/build.gradle index 7e57403..918f84c 100644 --- a/build.gradle +++ b/build.gradle @@ -21,6 +21,21 @@ allprojects { apply plugin: 'eclipse' } +buildscript { + repositories { + jcenter() + mavenCentral() + } + + dependencies { + classpath 'com.bmuschko:gradle-docker-plugin:2.6.1' + } +} + +repositories { + mavenCentral() +} + idea { project { languageLevel = '1.7' http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/docker/Dockerfile b/docker/Dockerfile index 18d4084..52ad7f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -43,7 +43,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh RUN sh /install-yarn.sh # Copy over myriad libraries -ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ +ADD ./libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ # Initialize hadoop confs with env vars http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/README.md ---------------------------------------------------------------------- diff --git a/docker/README.md b/docker/README.md index 8f00d64..7ce77ac 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,12 +1,14 @@ # Using Myriad with Docker # +Docker is the easiest way to from 0 to Yarn on Mesos within minutes. ## Building the Resource Manager Docker - -`./build-myriad.sh` will run the gradle scripts from the root myriad folder and compile all necessary libraries. +Run the following command: +```./gradlew buildRMDocker``` +This will build the ResourceManager from src and save the image as *mesos/myriad*. #Configuration Guide# -In order for the ResourceManager to operate correctly, you will need to provide 2 configuration files: +In order for the ResourceManager to operate correctly, you will need to provide 2 configuration files. These files will need to mounted from a directory into */myriad-conf* within the docker container. * [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml) * modified [yarn-site.xml](https://github.com/mesos/myriad/blob/phase1/docs/myriad-dev.md) @@ -17,19 +19,11 @@ In order for the ResourceManager to operate correctly, you will need to provide ```bash docker run --net=host --name='myriad-resourcemanager' -t \ -v /path/to/configs:/myriad-conf \ - -e HADOOP_NAMENODE="10.100.3.237:9000" \ - mesos/myriad-resourcemanager + mesos/myriad ``` #Environment Variables# - -* *HADOOP_NAMENODE* : *Required* -* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used by Myriad. - - -If you already had a working Vagrant instance, you will need to run `vagrant reload` in order to allow zookeeper and hdfs port-forwarding. - -Still having problems connecting to Zookeeper? Be sure that your zkServer and MesosMaster values are correct in the [myriad-config-default.yml](https://github.com/mesos/myriad/blob/phase1/myriad-scheduler/src/main/resources/myriad-config-default.yml) file. +* *ALTERNATE_HADOOP_URL* : Optional - Allows user to override the hadoop distribution used by Myriad. This will download the *.tar.gz file to be used as the hadoop distribution of choice for Myriad. --- <sub> http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/ResourceManager.dockerfile ---------------------------------------------------------------------- diff --git a/docker/ResourceManager.dockerfile b/docker/ResourceManager.dockerfile deleted file mode 100644 index c4448d7..0000000 --- a/docker/ResourceManager.dockerfile +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# Myriad - YARN Resource Manager -# -# VERSION 0.1.0 - -FROM debian -MAINTAINER Apache Myriad d...@myriad.incubator.apache.org - -ENV HADOOP_USER="yarn" -ENV HADOOP_VER="2.7.0" - -# Setup mesosphere repositories -RUN apt-get -y update -RUN apt-get install -y openjdk-7-jre-headless wget lsb-release -RUN apt-key adv --keyserver keyserver.ubuntu.com --recv E56151BF -RUN DISTRO=$(lsb_release -is | tr '[:upper:]' '[:lower:]') CODENAME=$(lsb_release -cs) && echo "deb http://repos.mesosphere.com/${DISTRO} ${CODENAME} main" | tee /etc/apt/sources.list.d/mesosphere.list -RUN apt-get -y update - -# Create the yarn user -ADD myriad-bin/create-user.sh /create-user.sh -RUN sh /create-user.sh - -# Install Mesos -RUN apt-get install -y mesos curl tar - -# Run local YARN install -ADD myriad-bin/install-yarn.sh /install-yarn.sh -RUN sh /install-yarn.sh - -# Copy over myriad libraries -ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ - -# Initialize hadoop confs with env vars -ADD myriad-bin/run-myriad.sh /run-myriad.sh -RUN mkdir /myriad-conf/ - -# Run the YARN resource manager -USER yarn -CMD sh /run-myriad.sh http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/build.gradle ---------------------------------------------------------------------- diff --git a/docker/build.gradle b/docker/build.gradle new file mode 100644 index 0000000..4f35494 --- /dev/null +++ b/docker/build.gradle @@ -0,0 +1,50 @@ +task wrapper(type: Wrapper) { + gradleVersion = '2.9' +} + +buildscript { + repositories { + jcenter() + mavenCentral() + } + + dependencies { + classpath 'com.bmuschko:gradle-docker-plugin:2.6.1' + } +} + +repositories { + mavenCentral() +} + +apply plugin: 'com.bmuschko.docker-remote-api' + +import com.bmuschko.gradle.docker.tasks.image.Dockerfile +import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage + +task copySchedulerJars(type:Copy){ + def subdir = new File(project.buildDir, "libs") + if( !subdir.exists() ) { subdir.mkdirs() } + from '../myriad-scheduler/build/libs' + into 'libs' +} + +task copyMyriadJars(dependsOn: ":myriad-scheduler:build") { + dependsOn(":myriad-executor:build") + dependsOn(copySchedulerJars) +} + +task buildRMDocker(type: DockerBuildImage) { + dependsOn(copyMyriadJars) + if (System.env.DOCKER_HOST) { + url = "$System.env.DOCKER_HOST".replace("tcp","https") + if (System.env.DOCKER_CERT_PATH) { + certPath = new File(System.env.DOCKER_CERT_PATH) + } + } + else { + url = 'unix:///var/run/docker.sock' + } + inputDir = file('.') + tag = 'mesos/myriad' +} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/docker/myriad-bin/create-user.sh ---------------------------------------------------------------------- diff --git a/docker/myriad-bin/create-user.sh b/docker/myriad-bin/create-user.sh index 9d6b2c2..c6c1bf6 100755 --- a/docker/myriad-bin/create-user.sh +++ b/docker/myriad-bin/create-user.sh @@ -27,7 +27,7 @@ export GROUP_UID=${GROUP_GID:='112'} # Add hduser user echo "Creating $HADOOP_USER user.." groupadd $HADOOP_GROUP -g ${GROUP_UID} -useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash +useradd $HADOOP_USER -g $HADOOP_GROUP -u ${USER_UID} -s /bin/bash -d /home/${HADOOP_USER} mkdir /home/${HADOOP_USER} chown -R $HADOOP_USER:$HADOOP_GROUP /home/${HADOOP_USER} http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/e1db4d8b/settings.gradle ---------------------------------------------------------------------- diff --git a/settings.gradle b/settings.gradle index 9af746c..8569bf5 100644 --- a/settings.gradle +++ b/settings.gradle @@ -29,3 +29,4 @@ rootProject.name = 'myriad' include "myriad-commons" include "myriad-scheduler" include "myriad-executor" +include "docker"
