This is an automated email from the ASF dual-hosted git repository.
sekikn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 75b0271 BIGTOP-3355. [Puppet] Make GPG check for repos a
configuration and default to true (#649)
75b0271 is described below
commit 75b02713566eb3603f486482215573274342d922
Author: Kengo Seki <[email protected]>
AuthorDate: Sun Jul 5 21:53:46 2020 +0900
BIGTOP-3355. [Puppet] Make GPG check for repos a configuration and default
to true (#649)
---
bigtop-deploy/puppet/hiera.yaml | 1 +
bigtop-deploy/puppet/hieradata/bigtop/repo.yaml | 5 ++
bigtop-deploy/puppet/manifests/bigtop_repo.pp | 101 +++++++++++++++---------
provisioner/docker/docker-hadoop.sh | 7 ++
provisioner/utils/setup-env-debian.sh | 3 +-
5 files changed, 78 insertions(+), 39 deletions(-)
diff --git a/bigtop-deploy/puppet/hiera.yaml b/bigtop-deploy/puppet/hiera.yaml
index b276006..848c550 100644
--- a/bigtop-deploy/puppet/hiera.yaml
+++ b/bigtop-deploy/puppet/hiera.yaml
@@ -5,3 +5,4 @@
- site
- "bigtop/%{hadoop_hiera_ha_path}"
- bigtop/cluster
+ - bigtop/repo
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/repo.yaml
b/bigtop-deploy/puppet/hieradata/bigtop/repo.yaml
new file mode 100644
index 0000000..48c8a4a
--- /dev/null
+++ b/bigtop-deploy/puppet/hieradata/bigtop/repo.yaml
@@ -0,0 +1,5 @@
+bigtop::bigtop_repo_gpg_check: true
+bigtop::bigtop_repo_apt_key: "BB95B97B18226C73CB2838D1DBBF9D42B7B4BD70"
+bigtop::bigtop_repo_yum_key_url: "https://downloads.apache.org/bigtop/KEYS";
+bigtop::bigtop_repo_default_version: "1.4.0"
+
diff --git a/bigtop-deploy/puppet/manifests/bigtop_repo.pp
b/bigtop-deploy/puppet/manifests/bigtop_repo.pp
index 3356112..44250f3 100644
--- a/bigtop-deploy/puppet/manifests/bigtop_repo.pp
+++ b/bigtop-deploy/puppet/manifests/bigtop_repo.pp
@@ -14,58 +14,83 @@
# limitations under the License.
class bigtop_repo {
+ $bigtop_repo_default_version = hiera("bigtop::bigtop_repo_default_version")
+ $bigtop_repo_gpg_check = hiera("bigtop::bigtop_repo_gpg_check", true)
+ $lower_os = downcase($operatingsystem)
+ $default_repo =
"http://repos.bigtop.apache.org/releases/${bigtop_repo_default_version}/${lower_os}/${operatingsystemmajrelease}/${architecture}";
+
case $::operatingsystem {
/(OracleLinux|Amazon|CentOS|Fedora|RedHat)/: {
- $default_repo =
"http://repos.bigtop.apache.org/releases/1.3.0/centos/7/x86_64";
$baseurls_array = any2array(hiera("bigtop::bigtop_repo_uri",
$default_repo))
- each ($baseurls_array) |$count, $baseurl| {
- yumrepo { "Bigtop_$count":
- baseurl => $baseurl,
- descr => "Bigtop packages",
- enabled => 1,
- gpgcheck => 0,
- priority => 10,
+ each($baseurls_array) |$count, $baseurl| {
+ notify { "Baseurl: $baseurl": }
+
+ if ($bigtop_repo_gpg_check) {
+ yumrepo { "Bigtop_$count":
+ baseurl => $baseurl,
+ descr => "Bigtop packages",
+ enabled => 1,
+ gpgcheck => 1,
+ gpgkey => hiera("bigtop::bigtop_repo_yum_key_url"),
+ priority => 10,
+ ensure => present,
+ }
+ } else {
+ yumrepo { "Bigtop_$count":
+ baseurl => $baseurl,
+ descr => "Bigtop packages",
+ enabled => 1,
+ gpgcheck => 0,
+ priority => 10,
+ ensure => present,
+ }
}
- Yumrepo<||> -> Package<||>
}
+ Yumrepo<||> -> Package<||>
}
/(Ubuntu|Debian)/: {
- include stdlib
- include apt
+ include stdlib
+ include apt
+ $baseurls_array = any2array(hiera("bigtop::bigtop_repo_uri",
$default_repo))
- $lower_os = downcase($operatingsystem)
- # We use code name such as trusty for Ubuntu instead of release version
in bigtop's binary convenience repos
- if ($operatingsystem == "Ubuntu") { $release = $lsbdistcodename } else
{ $release = $operatingsystemmajrelease }
- $default_repo =
"http://repos.bigtop.apache.org/releases/1.3.0/${lower_os}/${release}/x86_64";
- $baseurls_array = any2array(hiera("bigtop::bigtop_repo_uri",
$default_repo))
+ each($baseurls_array) |$count, $baseurl| {
+ notify { "Baseurl: $baseurl": }
+
+ apt::source { "Bigtop_$count":
+ location => $baseurl,
+ release => "bigtop",
+ repos => "contrib",
+ # BIGTOP-2796. Give Bigtop repo higher priority to solve zookeeper
package conflict probem on Ubuntu
+ pin => "900",
+ ensure => present,
+ }
+ }
- # I couldn't enforce the sequence -> anymore because of this
- # https://twitter.com/c0sin/status/875869783979196416
- apt::conf { "disable_keys":
- content => "APT::Get::AllowUnauthenticated 1;",
- ensure => present
- }
- each ($baseurls_array) |$count, $baseurl| {
- notify {"Baseurl: $baseurl" :}
- apt::source { "Bigtop_$count":
- location => $baseurl,
- release => "bigtop",
- repos => "contrib",
- # BIGTOP-2796. Give Bigtop repo higher priority to solve zookeeper
package conflict probem on Ubuntu
- pin => "900",
- ensure => present,
- }
- }
# It seems that calling update explicitely isn't needed because as far I
can see
# it is getting called automatically. Perhaps this was needed for older
versions?
- exec {'bigtop-apt-update':
- command => '/usr/bin/apt-get update'
- }
- Apt::Source<||> -> Exec['bigtop-apt-update'] -> Package<||>
+ exec { 'bigtop-apt-update':
+ command => '/usr/bin/apt-get update'
+ }
+
+ if ($bigtop_repo_gpg_check) {
+ apt::conf { "remove_disable_keys":
+ content => "APT::Get::AllowUnauthenticated 1;",
+ ensure => absent
+ }
+ apt::key { "add_key":
+ id => hiera("bigtop::bigtop_repo_apt_key"),
+ }
+ } else {
+ apt::conf { "disable_keys":
+ content => "APT::Get::AllowUnauthenticated 1;",
+ ensure => present
+ }
+ }
+ Apt::Conf<||> -> Apt::Key<||> -> Apt::Source<||> ->
Exec['bigtop-apt-update'] -> Package<||>
}
default: {
- notify{"WARNING: running on a neither yum nor apt platform -- make sure
Bigtop repo is setup": }
+ notify { "WARNING: running on a neither yum nor apt platform -- make
sure Bigtop repo is setup": }
}
}
}
diff --git a/provisioner/docker/docker-hadoop.sh
b/provisioner/docker/docker-hadoop.sh
index 5d902e3..087a1a0 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -97,6 +97,12 @@ create() {
fi
if [ -z ${enable_local_repo+x} ]; then
enable_local_repo=$(get-yaml-config enable_local_repo)
+ if [ $enable_local_repo == true ]; then
+ # When enabling local repo, set gpg check to false
+ gpg_check=false
+ else
+ gpg_check=true
+ fi
fi
generate-config "$hadoop_head_node" "$repo" "$components"
@@ -130,6 +136,7 @@ generate-config() {
bigtop::hadoop_head_node: $1
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
bigtop::bigtop_repo_uri: $2
+bigtop::bigtop_repo_gpg_check: $gpg_check
hadoop_cluster_node::cluster_components: $3
hadoop_cluster_node::cluster_nodes: [$node_list]
EOF
diff --git a/provisioner/utils/setup-env-debian.sh
b/provisioner/utils/setup-env-debian.sh
index 3d7ce8d..af44e05 100755
--- a/provisioner/utils/setup-env-debian.sh
+++ b/provisioner/utils/setup-env-debian.sh
@@ -38,7 +38,8 @@ Pin-Priority: 901
EOF
apt-get update
else
- apt-get install -y apt-transport-https
+ # Install gpg so that puppet apt module can fetch the gpg key
+ apt-get install -y apt-transport-https gnupg
echo "local apt = $enable_local_repo ; NOT Enabling local apt. Packages
will be pulled from remote..."
fi
