Repository: bigtop Updated Branches: refs/heads/BIGTOP-2982 [created] 11a9f21f0
BIGTOP-2982: POC provisioner based on docker-compose Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/30d345f6 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/30d345f6 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/30d345f6 Branch: refs/heads/BIGTOP-2982 Commit: 30d345f6e7e04d2034b5465c8b6a2b31bbe7b149 Parents: 51e751f Author: Olaf Flebbe <[email protected]> Authored: Fri Jan 5 22:33:42 2018 +0100 Committer: Olaf Flebbe <[email protected]> Committed: Tue Jan 9 20:37:47 2018 +0100 ---------------------------------------------------------------------- bigtop-deploy/puppet/manifests/bigtop_repo.pp | 2 + provisioner/compose/Dockerfile | 23 ++++++ provisioner/compose/README.md | 60 ++++++++++++++ provisioner/compose/docker-compose.yml | 95 ++++++++++++++++++++++ provisioner/compose/site.yaml | 19 +++++ 5 files changed, 199 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/30d345f6/bigtop-deploy/puppet/manifests/bigtop_repo.pp ---------------------------------------------------------------------- diff --git a/bigtop-deploy/puppet/manifests/bigtop_repo.pp b/bigtop-deploy/puppet/manifests/bigtop_repo.pp index ce17858..54fa596 100644 --- a/bigtop-deploy/puppet/manifests/bigtop_repo.pp +++ b/bigtop-deploy/puppet/manifests/bigtop_repo.pp @@ -14,6 +14,8 @@ # limitations under the License. class bigtop_repo { + include stdlib + case $::operatingsystem { /(OracleLinux|Amazon|CentOS|Fedora|RedHat)/: { $default_repo = "http://repos.bigtop.apache.org/releases/1.2.1/centos/7/x86_64"; http://git-wip-us.apache.org/repos/asf/bigtop/blob/30d345f6/provisioner/compose/Dockerfile ---------------------------------------------------------------------- diff --git a/provisioner/compose/Dockerfile b/provisioner/compose/Dockerfile new file mode 100644 index 0000000..ff09c00 --- /dev/null +++ b/provisioner/compose/Dockerfile @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM bigtop/puppet:trunk-centos-7 +ENV container docker + +COPY provisioner/utils/setup-env-centos.sh /tmp +RUN bash /tmp/setup-env-centos.sh && yum -y install bind-utils && mkdir /etc/puppet/hieradata +COPY bigtop-deploy/puppet/ /etc/puppet/ +RUN cp /etc/puppet/hiera.yaml /etc/hiera.yaml +COPY provisioner/compose/site.yaml /etc/puppet/hieradata/site.yaml http://git-wip-us.apache.org/repos/asf/bigtop/blob/30d345f6/provisioner/compose/README.md ---------------------------------------------------------------------- diff --git a/provisioner/compose/README.md b/provisioner/compose/README.md new file mode 100644 index 0000000..4ba0236 --- /dev/null +++ b/provisioner/compose/README.md @@ -0,0 +1,60 @@ +This is a proof of concept provisioner based on docker local clustering + +NAME RESOLUTION +=============== +In order to enable docker DNS we need a docker network +Create with (need only to be done once) +# docker network create --driver=bridge bigtop.org + +Setting the hostname to the service name enables us to have FQDN +of the hosts set correctly in /etc/hosts + +x.y.z.t headnode.bigtop.org headnode + +This is important since name resolution in /etc/nsswitch.conf is +hosts: files, dns + +The other members of the cluster datanode1 and datanode2 will be resolved by DNS(and headnode by DNS for these cluster nodes respectivly) + +External network enables us to have control over the domain name of the +cluster DNS + +VOLUMES +======== +In order to have performant I/O we use volumes + + +CREATING CLUSTER +================ +# docker-compose up -d + +This builds (if not already done) the initial container, containing all puppet configs. + +PROVISIONING +============ + +# docker-compose exec headnode puppet apply --parser=future /etc/puppet/manifests + +run puppet on headnode, install bigtop and config. dito for other nodes. Can be done in parallel + +# docker-compose exec datanode1 puppet apply --parser=future /etc/puppet/manifests +# docker-compose exec datanode2 puppet apply --parser=future /etc/puppet/manifests + +Cluster is now up and running + +UPDATING CONFIG +=============== + +If you are changing puppet scripts, run + +# docker-compose build + +Not sure if we are loosing HDFS data this way. + + +TEARING DOWN +============ + +docker-compose down -v +# since we are allocating volumes, remove them too + http://git-wip-us.apache.org/repos/asf/bigtop/blob/30d345f6/provisioner/compose/docker-compose.yml ---------------------------------------------------------------------- diff --git a/provisioner/compose/docker-compose.yml b/provisioner/compose/docker-compose.yml new file mode 100644 index 0000000..a5bfca0 --- /dev/null +++ b/provisioner/compose/docker-compose.yml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3' +services: + datanode1: + build: + context: ../.. + dockerfile: provisioner/compose/Dockerfile + command: /sbin/init + domainname: bigtop.org + hostname: datanode1 + container_name: datanode1 + networks: + bigtop.org: + aliases: + - datanode + volumes: + - /data/1 + - /data/2 + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /run + - /tmp + security_opt: + - seccomp=unconfined + cap_add: + - SYS_ADMIN + + datanode2: + build: + context: ../.. + dockerfile: provisioner/compose/Dockerfile + command: /sbin/init + domainname: bigtop.org + container_name: datanode2 + hostname: datanode2 + networks: + bigtop.org: + volumes: + - /data/1 + - /data/2 + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /run + - /tmp + security_opt: + - seccomp=unconfined + cap_add: + - SYS_ADMIN + + + + + headnode: + build: + context: ../.. + dockerfile: provisioner/compose/Dockerfile + command: /sbin/init + domainname: bigtop.org + hostname: headnode + container_name: headnode + networks: + - bigtop.org + volumes: + - /data/1 + - /data/2 + - /sys/fs/cgroup:/sys/fs/cgroup:ro + tmpfs: + - /run + - /tmp + security_opt: + - seccomp=unconfined + cap_add: + - SYS_ADMIN + + + + +networks: + bigtop.org: + external: + name: bigtop.org http://git-wip-us.apache.org/repos/asf/bigtop/blob/30d345f6/provisioner/compose/site.yaml ---------------------------------------------------------------------- diff --git a/provisioner/compose/site.yaml b/provisioner/compose/site.yaml new file mode 100644 index 0000000..8c65426 --- /dev/null +++ b/provisioner/compose/site.yaml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bigtop::hadoop_head_node: headnode.bigtop.org +hadoop::hadoop_storage_dirs: [/data/1, /data/2] +bigtop::bigtop_repo_uri: "http://repos.bigtop.apache.org/releases/1.2.1/centos/7/x86_64"; +hadoop_cluster_node::cluster_components: [hdfs, yarn, mapreduce]
