BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ced9142e Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ced9142e Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ced9142e Branch: refs/heads/BIGTOP-2320 Commit: ced9142ede5518c30ca1226460984ea59edfe4d9 Parents: 1d68396 Author: Konstantin Boudnik <[email protected]> Authored: Thu Feb 11 19:33:30 2016 +0300 Committer: Cos <[email protected]> Committed: Fri Mar 18 22:43:01 2016 -0500 ---------------------------------------------------------------------- bigtop_toolchain/manifests/installer.pp | 3 ++ bigtop_toolchain/manifests/libhdfs.pp | 45 ++++++++++++++++++++++++++++ bigtop_toolchain/manifests/libthrift.pp | 37 +++++++++++++++++++++++ bigtop_toolchain/manifests/packages.pp | 11 ++++--- bigtop_toolchain/manifests/protobuf.pp | 11 ++++--- 5 files changed, 99 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/ced9142e/bigtop_toolchain/manifests/installer.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/installer.pp b/bigtop_toolchain/manifests/installer.pp index 246fcaf..d29a222 100644 --- a/bigtop_toolchain/manifests/installer.pp +++ b/bigtop_toolchain/manifests/installer.pp @@ -20,6 +20,9 @@ class bigtop_toolchain::installer { include bigtop_toolchain::gradle include bigtop_toolchain::protobuf include bigtop_toolchain::packages +## The hack workaround until such times when the lib is embedded into Hawq + include bigtop_toolchain::libhdfs + include bigtop_toolchain::libthrift include bigtop_toolchain::env include bigtop_toolchain::user http://git-wip-us.apache.org/repos/asf/bigtop/blob/ced9142e/bigtop_toolchain/manifests/libhdfs.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/libhdfs.pp b/bigtop_toolchain/manifests/libhdfs.pp new file mode 100644 index 0000000..4698108 --- /dev/null +++ b/bigtop_toolchain/manifests/libhdfs.pp @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class bigtop_toolchain::libhdfs { + + include bigtop_toolchain::packages + + + case $operatingsystem { + /(?i:(centos|fedora|amazon))/: { + + $apache_prefix = nearest_apache_mirror() + $hackrepourl = "https://bintray.com/artifact/download/wangzw/rpm/centos7/x86_64"; + $libhdfs = "libhdfs3-2.2.31-1.el7.centos.x86_64.rpm" + $libhdfs_dev = "libhdfs3-devel-2.2.31-1.el7.centos.x86_64.rpm" + + exec {"download": + path => "/usr/bin", + cwd => "/usr/src", + command => "curl -L $hackrepourl/$libhdfs -o $libhdfs ; curl -L $hackrepourl/$libhdfs_dev -o $libhdfs_dev", + creates => "/usr/src/$libhdfs, /usr/src/$libhdfs_dev", + } + + exec {"install": + path => "/usr/bin", + cwd => "/usr/src", + command => "yum install -y $libhdfs $libhdfs_dev", + require => [ Exec[ 'download' ], Package[ $packages::pkgs ] ], + } + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ced9142e/bigtop_toolchain/manifests/libthrift.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/libthrift.pp b/bigtop_toolchain/manifests/libthrift.pp new file mode 100644 index 0000000..bc1abf3 --- /dev/null +++ b/bigtop_toolchain/manifests/libthrift.pp @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# eee the License for the specific language governing permissions and +# limitations under the License. + +class bigtop_toolchain::libthrift { + + include bigtop_toolchain::packages + + case $operatingsystem { + /Ubuntu|Debian/: { + include apt + apt::conf { "disable_keys": + content => "APT::Get::AllowUnauthenticated 1;", + ensure => present + } + apt::source { "libthrift": + location => "http://dl.bintray.com/c0s/deb";, + release => "trusty", + repos => "contrib", + ensure => present, + } + Apt::Source<||> -> Exec['apt_update'] -> Package<||> + + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/ced9142e/bigtop_toolchain/manifests/packages.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp index 26959e2..bbdc764 100644 --- a/bigtop_toolchain/manifests/packages.pp +++ b/bigtop_toolchain/manifests/packages.pp @@ -22,16 +22,19 @@ class bigtop_toolchain::packages { } else { $mysql_devel="mariadb-devel" } - $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ] } - /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ] + $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel", "thrift-devel" ] + exec { '/usr/bin/yum install -y epel-release': + } -> Package <| |> + } + /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel", "boost-devel", "xfsprogs-devel", "libuuid-devel" ] # fix package dependencies: BIGTOP-2120 and BIGTOP-2152 exec { '/usr/bin/zypper remove -y krb5-mini': } -> exec {'/usr/bin/zypper install -y libopenssl-devel': } -> Package <| |> } - Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] } - /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev", "libboost-regex-dev", "xfslibs-dev" ] + Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] } + /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl4-openssl-dev", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "libapr1-dev", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libbz2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "uuid", "libgsasl7-dev", "libevent-dev", "libjson-c-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libreadline-dev", "libsnappy-dev", "libboost-regex-dev", "libboost-system-dev", "libboost-thread-dev", "libthrift0", "libthrift-dev", "xfslibs-dev" ] exec { "apt-update": command => "/usr/bin/apt-get update" http://git-wip-us.apache.org/repos/asf/bigtop/blob/ced9142e/bigtop_toolchain/manifests/protobuf.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/protobuf.pp b/bigtop_toolchain/manifests/protobuf.pp index c71dec2..53d539b 100644 --- a/bigtop_toolchain/manifests/protobuf.pp +++ b/bigtop_toolchain/manifests/protobuf.pp @@ -21,13 +21,15 @@ class bigtop_toolchain::protobuf { /Ubuntu|Debian/: { case $architecture { 'amd64' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5585371/+files/"; + $url1= "http://launchpadlibrarian.net/166035876/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb"; + $url2= "http://launchpadlibrarian.net/166035877/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb"; $arch= "amd64" } 'ppc64le' : { $url = "https://launchpad.net/ubuntu/+source/protobuf/2.5.0-9ubuntu1/+build/5604345/+files"; $arch= "ppc64el" } } } } - + case $operatingsystem{ /Ubuntu|Debian/: { $libprotobuf8 = "libprotobuf8_2.5.0-9ubuntu1_$arch.deb" @@ -35,13 +37,14 @@ class bigtop_toolchain::protobuf { $protobuf_compiler = "protobuf-compiler_2.5.0-9ubuntu1_$arch.deb" exec { "download protobuf": + path => "/usr/bin", cwd => "/usr/src", - command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler", - creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler" ] + command => "/usr/bin/curl -L $url/$libprotobuf8 -o $libprotobuf8; /usr/bin/curl -L $url/$libprotoc8 -o $libprotoc8; /usr/bin/curl -L $url/$protobuf_compiler -o $protobuf_compiler; curl -L $url1 -o libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb; curl -L $url2 -o libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb", + creates => [ "/usr/src/$libprotobuf8", "/usr/src/$libprotoc8", "/usr/src/$protobuf_compiler", "/usr/src/libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb", "/usr/src/libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb" ] } exec { "install protobuf": cwd => "/usr/src", - command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler", + command => "/usr/bin/dpkg -i $libprotobuf8 $libprotoc8 $protobuf_compiler libprotobuf-lite8_2.5.0-9ubuntu1_amd64.deb libprotobuf-dev_2.5.0-9ubuntu1_amd64.deb", require => EXEC["download protobuf"], } }
