Repository: bigtop Updated Branches: refs/heads/master 31de2c517 -> 639348c80
BIGTOP-2615. Provide a tool to build pseudo cluster docker images Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/639348c8 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/639348c8 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/639348c8 Branch: refs/heads/master Commit: 639348c809f46bbc8585a96f6d35d04077880232 Parents: 31de2c5 Author: Evans Ye <[email protected]> Authored: Fri Dec 2 23:01:50 2016 +0800 Committer: Evans Ye <[email protected]> Committed: Tue Jan 17 01:33:58 2017 +0800 ---------------------------------------------------------------------- docker/sandbox/README.md | 93 ++++++++ docker/sandbox/build.sh | 215 +++++++++++++++++++ docker/sandbox/docker-compose.yml | 23 ++ docker/sandbox/sandbox-env.sh | 24 +++ .../sandbox/site.yaml.template.centos-6_hadoop | 20 ++ .../sandbox/site.yaml.template.debian-8_hadoop | 20 ++ docker/sandbox/startup.sh | 45 ++++ 7 files changed, 440 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/README.md ---------------------------------------------------------------------- diff --git a/docker/sandbox/README.md b/docker/sandbox/README.md new file mode 100644 index 0000000..89c9e5e --- /dev/null +++ b/docker/sandbox/README.md @@ -0,0 +1,93 @@ +# Bigtop Docker Sandbox + +A tool to build and run big data pseudo cluster using Docker. + +## How to run + +* Make sure you have Docker installed. We've tested this using [Docker for Mac](https://docs.docker.com/docker-for-mac/) + +* Currently supported OS list: + + * centos-6 + * debian-8 + * ubuntu-14.04 + +* Running Hadoop + +``` +docker run -ti --privileged -p 50070:50070 bigtop/sandbox:centos-6_hadoop +``` + +* Running Spark (Standalone mode) + +``` +docker run -ti --privileged -p 8080:8080 bigtop/sandbox:debian-8_spark +``` + +* Running Hadoop + HBase + +``` +docker run -ti --privileged -p 50070:50070 -p 60010:60010 bigtop/sandbox:ubuntu-14.04_hbase +``` + +## How to build + +### Examples + +* Build sandbox image that has Hadoop provisioned + +``` +./build.sh -a bigtop -o debian-8 -c hadoop +``` + +* Build sandbox image that has Hadoop and Spark provisioned + +``` +./build.sh -a bigtop -o debian-8 -c "hadoop, spark" +``` + +* Build sandbox image that has Hadoop and HBase provisioned + +``` +./build.sh -a bigtop -o debian-8 -c "hadoop, yarn, hbase" +``` + +### Change the repository of packages + +* Change the repository to Bigtop's nightly centos-6 repo + +``` +export REPO=http://ci.bigtop.apache.org:8080/job/Bigtop-trunk-repos/BUILD_ENVIRONMENTS=centos-6%2Clabel=docker-slave-06//ws/output +./build.sh -a bigtop -o centos-6 -c "hadoop, spark, ignite" +``` + +### Customize your Hadoop stack + +* Edit *site.yaml.template.centos-6_hadoop* to create your own prefered stack + +``` +cp site.yaml.template.centos-6_hadoop site.yaml.template.centos-6_hadoop_ignite +vim site.yaml.template.centos-6_hadoop_ignite +``` + +* Add ignite in *hadoop_cluster_node::cluster_components* array and leave the other unchanged + +``` +... +hadoop_cluster_node::cluster_components: [hadoop, yarn, ignite] +... +``` + +* Build + +``` +./build.sh -a bigtop -o centos-6 -f site.yaml.template.centos-6_hadoop_ignite -t my_ignite_stack +``` + +## Known issues + +### Fail to start daemons using systemd + +Since systemd requires CAP_SYS_ADMIN, currently any OS using systemd can not successfully started up daemons during image build time. + +Daemons can be brought up only if --privileged specified using docker run command. http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/build.sh ---------------------------------------------------------------------- diff --git a/docker/sandbox/build.sh b/docker/sandbox/build.sh new file mode 100755 index 0000000..8936fd0 --- /dev/null +++ b/docker/sandbox/build.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source sandbox-env.sh + +build() { + # prepare puppet recipes + cp -r ../../bigtop-deploy/puppet bigtop-puppet + + # docker build + docker-compose build --force-rm --no-cache --pull + + if [ $? -eq 0 ]; then + echo "-------------------------------------------------" + echo "Image $ACCOUNT/sandbox:$TAG built" + echo "-------------------------------------------------" + fi +} + +cleanup() { + rm -rf bigtop-puppet site.yaml.template Dockerfile +} + +generate_config() { + cat > site.yaml.template << EOF +bigtop::hadoop_head_node: ${HEAD_NODE:-"head.node.fqdn"} +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: ${REPO} +hadoop_cluster_node::cluster_components: [${COMPONENTS}] +bigtop::jdk_package_name: ${JDK} +EOF +} + +generate_dockerfile() { + cat > Dockerfile << EOF +FROM bigtop/puppet:${OS} +ADD startup.sh /startup.sh +ADD bigtop-puppet /bigtop-puppet +ADD bigtop-puppet/hiera.yaml /etc/puppet/hiera.yaml +ADD bigtop-puppet/hieradata /etc/puppet/hieradata +ADD site.yaml.template /etc/puppet/hieradata/site.yaml.template +RUN /startup.sh --bootstrap +CMD /startup.sh --foreground +EOF +} + +generate_tag() { + if [ -z "$TAG" ]; then + TAG="${OS}_`echo ${COMPONENTS//,/_} | tr -d ' '`" + fi +} + +detect_jdk() { + for RPM in ${RPMS[*]}; do + [[ $OS == $RPM ]] && JDK=$RPM_JDK + done + for DEB in ${DEBS[*]}; do + [[ $OS == $DEB ]] && JDK=$DEB_JDK + done +} + +detect_repo() { + OS_WITH_CODE_NAME=${OS/ubuntu-14.04/ubuntu-trusty} + REPO=${REPO:-"http://bigtop-repos.s3.amazonaws.com/releases/${BIGTOP_VERSION}/${OS_WITH_CODE_NAME/-//}/x86_64"} +} + +image_config_validator() { + invalid=1 + if [ -z "$ACCOUNT" ]; then + echo "account unset!" + invalid=0 + fi + if [ -z "$OS" ]; then + echo "operating system unset!" + invalid=0 + fi + if [ -z "$TAG" ]; then + echo "tag unset!" + invalid=0 + fi + if [ $invalid -eq 0 ]; then + usage + fi +} + +deploy_config_validator() { + invalid=1 + if [ -z "$REPO" ]; then + echo "repository unset!" + invalid=0 + fi + if [ -z "$JDK" ]; then + echo "jdk unset!" + invalid=0 + fi + if [ -z "$COMPONENTS" ]; then + echo "components unset!" + invalid=0 + fi + if [ $invalid -eq 0 ]; then + usage + fi +} + +show_image_configs() { + echo "-------------------------------------------------" + echo "IMAGE CONFIGS:" + echo "ACCOUNT $ACCOUNT" + echo "OS $OS" + echo "TAG $TAG" + echo "IMAGE $ACCOUNT/sandbox:$TAG" + echo "-------------------------------------------------" +} + +show_deploy_configs() { + echo "-------------------------------------------------" + echo "DEPLOY CONFIGS:" + echo "REPOSITORY $REPO" + echo "COMPONENTS $COMPONENTS" + echo "JDK $JDK" + echo "-------------------------------------------------" +} + +usage() { + echo "usage: $PROG args" + echo " -a, --account Specify account name for image." + echo " -c, --components Specify components to build." + echo " You need to specify a comma separated, quoted string." + echo " For example: --components \"hadoop, yarn\"" + echo " -f, --file Specify a written site.yaml config file." + echo " -o, --operating-system Specify an OS from Bigtop supported OS list." + echo " RPM base: ${RPMS[*]}" + echo " DEB base: ${DEBS[*]}" + echo " -t, --tag Specify tag for image." + echo " If not specified, this will be auto filled by specified OS and components." + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + -a|--account) + if [ $# -lt 2 ]; then + usage + fi + ACCOUNT=$2 + shift 2;; + -c|--components) + if [ $# -lt 2 ]; then + usage + fi + COMPONENTS=$2 + shift 2;; + -f|--file) + if [ $# -lt 2 ]; then + usage + fi + FILE=$2 + shift 2;; + -o|--operating-system) + if [ $# -lt 2 ]; then + usage + fi + OS=$2 + shift 2;; + -t|--tag) + if [ $# -lt 2 ]; then + usage + fi + TAG=$2 + shift 2;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done + +cleanup +generate_tag +image_config_validator +show_image_configs + +if [ -z "$FILE" ]; then + detect_jdk + detect_repo + deploy_config_validator + generate_config + show_deploy_configs +else + if [ -f "$FILE" ]; then + cp -vfr $FILE site.yaml.template + else + echo "$FILE not exist!" + exit 1 + fi +fi + +export ACCOUNT +export TAG +generate_dockerfile +build +cleanup http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/docker-compose.yml ---------------------------------------------------------------------- diff --git a/docker/sandbox/docker-compose.yml b/docker/sandbox/docker-compose.yml new file mode 100644 index 0000000..0a4ff0a --- /dev/null +++ b/docker/sandbox/docker-compose.yml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2' + +services: + sandbox: + build: + context: . + dockerfile: Dockerfile + image: ${ACCOUNT}/sandbox:${TAG} http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/sandbox-env.sh ---------------------------------------------------------------------- diff --git a/docker/sandbox/sandbox-env.sh b/docker/sandbox/sandbox-env.sh new file mode 100644 index 0000000..6eb5e54 --- /dev/null +++ b/docker/sandbox/sandbox-env.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +RPMS=(centos-6 centos-7 fedora-20) +DEBS=(debian-8 ubuntu-14.04) + +RPM_JDK=java-1.7.0-openjdk-devel.x86_64 +DEB_JDK=openjdk-7-jdk + +BIGTOP_VERSION=1.1.0 http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/site.yaml.template.centos-6_hadoop ---------------------------------------------------------------------- diff --git a/docker/sandbox/site.yaml.template.centos-6_hadoop b/docker/sandbox/site.yaml.template.centos-6_hadoop new file mode 100644 index 0000000..71a7c33 --- /dev/null +++ b/docker/sandbox/site.yaml.template.centos-6_hadoop @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bigtop::hadoop_head_node: "head.node.fqdn" +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64 +hadoop_cluster_node::cluster_components: [hadoop, yarn] +bigtop::jdk_package_name: java-1.7.0-openjdk-devel.x86_64 http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/site.yaml.template.debian-8_hadoop ---------------------------------------------------------------------- diff --git a/docker/sandbox/site.yaml.template.debian-8_hadoop b/docker/sandbox/site.yaml.template.debian-8_hadoop new file mode 100644 index 0000000..f2436ba --- /dev/null +++ b/docker/sandbox/site.yaml.template.debian-8_hadoop @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bigtop::hadoop_head_node: "head.node.fqdn" +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64 +hadoop_cluster_node::cluster_components: [hadoop, yarn] +bigtop::jdk_package_name: openjdk-7-jdk http://git-wip-us.apache.org/repos/asf/bigtop/blob/639348c8/docker/sandbox/startup.sh ---------------------------------------------------------------------- diff --git a/docker/sandbox/startup.sh b/docker/sandbox/startup.sh new file mode 100755 index 0000000..ff38894 --- /dev/null +++ b/docker/sandbox/startup.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +init() { + echo "`facter ipaddress` `facter fqdn`" >> /etc/hosts + cp /etc/puppet/hieradata/site.yaml.template /etc/puppet/hieradata/site.yaml + sed -i -e "s/head.node.fqdn/`facter fqdn`/g" /etc/puppet/hieradata/site.yaml + puppet apply --parser future --modulepath=/bigtop-puppet/modules:/etc/puppet/modules /bigtop-puppet/manifests +} + +usage() { + echo "usage: $PROG args" + echo " -f, --foreground Running foreground." + echo " -b, --build Bootstrap the stack." + exit 1 +} + +while [ $# -gt 0 ]; do + case "$1" in + -f|--foreground) + init + sleep infinity + shift;; + -b|--bootstrap) + init + shift;; + *) + echo "Unknown argument: '$1'" 1>&2 + usage;; + esac +done
