Repository: bigtop Updated Branches: refs/heads/master 639348c80 -> 182eafef2
BIGTOP-2668. Polish and update Docker Provisioner configurations Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/182eafef Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/182eafef Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/182eafef Branch: refs/heads/master Commit: 182eafef28c3247218b068e764d032233461d5b4 Parents: 639348c Author: Evans Ye <[email protected]> Authored: Sun Jan 15 03:41:29 2017 +0800 Committer: Evans Ye <[email protected]> Committed: Sat Jan 21 11:51:48 2017 +0800 ---------------------------------------------------------------------- provisioner/docker/README.md | 13 ++++----- provisioner/docker/config.yaml | 33 +---------------------- provisioner/docker/config_centos6.yaml | 25 +++++++++++++++++ provisioner/docker/config_centos7.yaml | 13 +++------ provisioner/docker/config_debian.yaml | 32 ---------------------- provisioner/docker/config_debian8.yaml | 25 +++++++++++++++++ provisioner/docker/config_ubuntu.yaml | 32 ---------------------- provisioner/docker/config_ubuntu_trusty.yaml | 25 +++++++++++++++++ provisioner/docker/docker-compose.yml | 1 + provisioner/docker/docker-hadoop.sh | 3 ++- 10 files changed, 87 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/README.md ---------------------------------------------------------------------- diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md index 8a8b8ee..ef52ff1 100644 --- a/provisioner/docker/README.md +++ b/provisioner/docker/README.md @@ -123,20 +123,17 @@ usage: docker-hadoop.sh [-C file ] args ``` docker: - memory_size: "2048" + memory_limit: "2g" ``` -2) Use different host ports mapping for web UIs +2) Enable local repository -``` -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" +If you've built packages using local cloned bigtop and produced the apt/yum repo, set the following to true to deploy those packages: ``` -Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed - +enable_local_repo = true +``` ##Configure Apache Hadoop ecosystem components * Choose the ecosystem you want to be deployed by modifying components in config.yaml http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml deleted file mode 100644 index 86f35e7..0000000 --- a/provisioner/docker/config.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:centos-6" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64"; -distro: centos -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "java-1.7.0-openjdk-devel.x86_64" diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml new file mode 120000 index 0000000..776a8f1 --- /dev/null +++ b/provisioner/docker/config.yaml @@ -0,0 +1 @@ +config_centos7.yaml \ No newline at end of file http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_centos6.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos6.yaml b/provisioner/docker/config_centos6.yaml new file mode 100644 index 0000000..54ca86c --- /dev/null +++ b/provisioner/docker/config_centos6.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:centos-6" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/6/x86_64"; +distro: centos +components: [hadoop, yarn] +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "java-1.7.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_centos7.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml index 3610a8f..44920f4 100644 --- a/provisioner/docker/config_centos7.yaml +++ b/provisioner/docker/config_centos7.yaml @@ -14,19 +14,12 @@ # limitations under the License. docker: - memory_size: "4096" - image: "bigtop/deploy:centos-7" + memory_limit: "4g" + image: "bigtop/puppet:centos-7" -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"; +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/centos/7/x86_64"; distro: centos components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" enable_local_repo: false smoke_test_components: [mapreduce, pig] jdk: "java-1.7.0-openjdk-devel.x86_64" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_debian.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian.yaml deleted file mode 100644 index f6b11ae..0000000 --- a/provisioner/docker/config_debian.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:debian-8" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64"; -distro: debian -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_debian8.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian8.yaml b/provisioner/docker/config_debian8.yaml new file mode 100644 index 0000000..2ed8946 --- /dev/null +++ b/provisioner/docker/config_debian8.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:debian-8" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/debian/8/x86_64"; +distro: debian +components: [hadoop, yarn] +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_ubuntu.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu.yaml deleted file mode 100644 index e4ea6f3..0000000 --- a/provisioner/docker/config_ubuntu.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_size: "4096" - image: "bigtop/deploy:ubuntu-14.04" - -boot2docker: - memory_size: "4096" - number_cpus: "1" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64"; -distro: debian -components: [hadoop, yarn] -namenode_ui_port: "50070" -yarn_ui_port: "8088" -hbase_ui_port: "60010" -enable_local_repo: false -smoke_test_components: [mapreduce, pig] -jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/config_ubuntu_trusty.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu_trusty.yaml b/provisioner/docker/config_ubuntu_trusty.yaml new file mode 100644 index 0000000..4ac83dc --- /dev/null +++ b/provisioner/docker/config_ubuntu_trusty.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:ubuntu-14.04" + +repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.1.0/ubuntu/trusty/x86_64"; +distro: debian +components: [hadoop, yarn] +enable_local_repo: false +smoke_test_components: [mapreduce, pig] +jdk: "openjdk-7-jdk" http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/docker-compose.yml ---------------------------------------------------------------------- diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml index df107ae..cee849e 100644 --- a/provisioner/docker/docker-compose.yml +++ b/provisioner/docker/docker-compose.yml @@ -18,6 +18,7 @@ bigtop: command: /sbin/init domainname: bigtop.apache.org privileged: true + mem_limit: ${MEM_LIMIT} volumes: - ../../:/bigtop-home - ./config/hiera.yaml:/etc/puppet/hiera.yaml http://git-wip-us.apache.org/repos/asf/bigtop/blob/182eafef/provisioner/docker/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index d40bd8b..05b712c 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -43,7 +43,6 @@ create() { mkdir -p config/hieradata 2> /dev/null echo > ./config/hiera.yaml echo > ./config/hosts - export DOCKER_IMAGE=$(get-yaml-config docker image) # Startup instances docker-compose -p $PROVISION_ID scale bigtop=$1 @@ -193,6 +192,8 @@ fi if [ -n "$PROVISION_ID" ]; then NODES=(`docker-compose -p $PROVISION_ID ps -q`) fi +export DOCKER_IMAGE=$(get-yaml-config docker image) +export MEM_LIMIT=$(get-yaml-config docker memory_limit) while [ $# -gt 0 ]; do case "$1" in
