Repository: bigtop Updated Branches: refs/heads/branch-1.3 980172cd2 -> 7f701affa
BIGTOP-3089: Update provision config files with changes in 1.3.0 release v1.3.0 supports: centos-7, fedora-26, debian-9, ubuntu-16.04, opensuse-42.3 config files in provisioner/docker should be updated to sync with these changes Now config files contain ARCH variable to automatically load arch-dependent repo. And docker-hadoop.sh will load correct puppet image by detecting CPU arch. This makes only one config file is needed for a certain distro acrossing different architectures. This closes #402 Change-Id: Ic5d66cb837e78908fcf19c2e55f819db28866a55 Signed-off-by: Jun He <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/cd43e324 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/cd43e324 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/cd43e324 Branch: refs/heads/branch-1.3 Commit: cd43e324bb0daa463d8337d74389c2edf6530924 Parents: 980172c Author: Jun He <[email protected]> Authored: Wed Oct 3 20:52:33 2018 +0800 Committer: Jun He <[email protected]> Committed: Fri Oct 5 01:24:23 2018 +0000 ---------------------------------------------------------------------- provisioner/docker/config_centos-6.yaml | 24 -------------------- provisioner/docker/config_centos-7-aarch64.yaml | 24 -------------------- provisioner/docker/config_centos-7.yaml | 2 +- provisioner/docker/config_debian-8-aarch64.yaml | 24 -------------------- provisioner/docker/config_debian-8.yaml | 24 -------------------- provisioner/docker/config_debian-9.yaml | 24 ++++++++++++++++++++ provisioner/docker/config_fedora-26.yaml | 24 ++++++++++++++++++++ .../docker/config_ubuntu-16.04-aarch64.yaml | 24 -------------------- provisioner/docker/config_ubuntu-16.04.yaml | 2 +- provisioner/docker/docker-hadoop.sh | 10 +++++++- 10 files changed, 59 insertions(+), 123 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_centos-6.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos-6.yaml b/provisioner/docker/config_centos-6.yaml deleted file mode 100644 index fbd6a83..0000000 --- a/provisioner/docker/config_centos-6.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:trunk-centos-6" - -repo: "http://repos.bigtop.apache.org/releases/1.2.1/centos/6/x86_64"; -distro: centos -components: [hdfs, yarn, mapreduce] -enable_local_repo: false -smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_centos-7-aarch64.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos-7-aarch64.yaml b/provisioner/docker/config_centos-7-aarch64.yaml deleted file mode 100644 index 09266f4..0000000 --- a/provisioner/docker/config_centos-7-aarch64.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:trunk-centos-7-aarch64" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/centos/7/aarch64"; -distro: centos -components: [hdfs, yarn, mapreduce] -enable_local_repo: false -smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_centos-7.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_centos-7.yaml b/provisioner/docker/config_centos-7.yaml index cff8c42..49f86ae 100644 --- a/provisioner/docker/config_centos-7.yaml +++ b/provisioner/docker/config_centos-7.yaml @@ -17,7 +17,7 @@ docker: memory_limit: "4g" image: "bigtop/puppet:trunk-centos-7" -repo: "http://repos.bigtop.apache.org/releases/1.3.0/centos/7/x86_64"; +repo: "http://repos.bigtop.apache.org/releases/1.3.0/centos/7/$basearch"; distro: centos components: [hdfs, yarn, mapreduce] enable_local_repo: false http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_debian-8-aarch64.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian-8-aarch64.yaml b/provisioner/docker/config_debian-8-aarch64.yaml deleted file mode 100644 index 3459391..0000000 --- a/provisioner/docker/config_debian-8-aarch64.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:trunk-debian-8-aarch64" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/debian/8-aarch64/aarch64"; -distro: debian -components: [hdfs, yarn, mapreduce] -enable_local_repo: false -smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_debian-8.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian-8.yaml b/provisioner/docker/config_debian-8.yaml deleted file mode 100644 index 1dbecbe..0000000 --- a/provisioner/docker/config_debian-8.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:trunk-debian-8" - -repo: "http://repos.bigtop.apache.org/releases/1.2.1/debian/8/x86_64"; -distro: debian -components: [hdfs, yarn, mapreduce] -enable_local_repo: false -smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_debian-9.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_debian-9.yaml b/provisioner/docker/config_debian-9.yaml new file mode 100644 index 0000000..1818f2d --- /dev/null +++ b/provisioner/docker/config_debian-9.yaml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:trunk-debian-9" + +repo: "http://repos.bigtop.apache.org/releases/1.3.0/debian/9/$(ARCH)" +distro: debian +components: [hdfs, yarn, mapreduce] +enable_local_repo: false +smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_fedora-26.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_fedora-26.yaml b/provisioner/docker/config_fedora-26.yaml new file mode 100644 index 0000000..8881c14 --- /dev/null +++ b/provisioner/docker/config_fedora-26.yaml @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + memory_limit: "4g" + image: "bigtop/puppet:trunk-fedora-26" + +repo: "http://repos.bigtop.apache.org/releases/1.3.0/fedora/26/$basearch"; +distro: centos +components: [hdfs, yarn, mapreduce] +enable_local_repo: false +smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_ubuntu-16.04-aarch64.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu-16.04-aarch64.yaml b/provisioner/docker/config_ubuntu-16.04-aarch64.yaml deleted file mode 100644 index 0a89475..0000000 --- a/provisioner/docker/config_ubuntu-16.04-aarch64.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -docker: - memory_limit: "4g" - image: "bigtop/puppet:trunk-ubuntu-16.04-aarch64" - -repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.2.0/ubuntu/16.04/x86_64"; -distro: debian -components: [hdfs, yarn, mapreduce] -enable_local_repo: true -smoke_test_components: [hdfs, yarn, mapreduce] http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/config_ubuntu-16.04.yaml ---------------------------------------------------------------------- diff --git a/provisioner/docker/config_ubuntu-16.04.yaml b/provisioner/docker/config_ubuntu-16.04.yaml index e5f7d8f..64cace9 100644 --- a/provisioner/docker/config_ubuntu-16.04.yaml +++ b/provisioner/docker/config_ubuntu-16.04.yaml @@ -17,7 +17,7 @@ docker: memory_limit: "4g" image: "bigtop/puppet:trunk-ubuntu-16.04" -repo: "http://repos.bigtop.apache.org/releases/1.3.0/ubuntu/16.04/x86_64"; +repo: "http://repos.bigtop.apache.org/releases/1.3.0/ubuntu/16.04/$(ARCH)" distro: debian components: [hdfs, yarn, mapreduce] enable_local_repo: false http://git-wip-us.apache.org/repos/asf/bigtop/blob/cd43e324/provisioner/docker/docker-hadoop.sh ---------------------------------------------------------------------- diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh index 203fc44..3300c93 100755 --- a/provisioner/docker/docker-hadoop.sh +++ b/provisioner/docker/docker-hadoop.sh @@ -43,7 +43,15 @@ create() { mkdir -p config/hieradata 2> /dev/null echo > ./config/hiera.yaml echo > ./config/hosts - export DOCKER_IMAGE=$(get-yaml-config docker image) + # set correct image name based on running architecture + image_name=$(get-yaml-config docker image) + running_arch=$(uname -m) + if [ "x86_64" == ${running_arch} ]; then + image_name=${image_name} + else + image_name=${image_name}-${running_arch} + fi + export DOCKER_IMAGE=${image_name} export MEM_LIMIT=$(get-yaml-config docker memory_limit) # Startup instances
