BIGTOP-2323. Build slaves toolchain needs to be updated for HAWQ libhdfs3-devel thrift-devel epel-release is needed as well
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a5243a01 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a5243a01 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a5243a01 Branch: refs/heads/BIGTOP-2320 Commit: a5243a013bd8107673aaac0e768f371ec3ce151b Parents: a5e67c1 Author: Konstantin Boudnik <[email protected]> Authored: Thu Feb 11 19:33:30 2016 +0300 Committer: Konstantin Boudnik <[email protected]> Committed: Thu Feb 25 14:48:20 2016 -0800 ---------------------------------------------------------------------- bigtop-packages/src/common/hawq/do-component-build | 14 ++++++++++++++ bigtop-packages/src/common/hawq/install_hawq.sh | 5 ++++- bigtop_toolchain/manifests/packages.pp | 8 ++++---- 3 files changed, 22 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5243a01/bigtop-packages/src/common/hawq/do-component-build ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hawq/do-component-build b/bigtop-packages/src/common/hawq/do-component-build index 07934b6..82436a5 100755 --- a/bigtop-packages/src/common/hawq/do-component-build +++ b/bigtop-packages/src/common/hawq/do-component-build @@ -34,6 +34,20 @@ cd ${CWD} ldconfig $CWD/libyarn/lib ## libyarn should be available in the system at this point +## build libhdfs3 +wget https://github.com/PivotalRD/libhdfs3/archive/v2.2.31.tar.gz +tar zxf v2.2.31.tar.gz +cd libhdfs3-2.2.31 +mkdir -p build +cd build +../bootstrap +make rpm-package +## need to run diff target for deb and rpm. Figure out how +cd ${CWD} +# now something like +# rpm -ivh rpms/RPMS/`uname -p`/*rpm +## libhdfs3 libhdfs3-devel should be available now + echo "Running the native build" date; ./configure --prefix=${CWD}/hawq \ http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5243a01/bigtop-packages/src/common/hawq/install_hawq.sh ---------------------------------------------------------------------- diff --git a/bigtop-packages/src/common/hawq/install_hawq.sh b/bigtop-packages/src/common/hawq/install_hawq.sh index df9945a..d1b7a59 100755 --- a/bigtop-packages/src/common/hawq/install_hawq.sh +++ b/bigtop-packages/src/common/hawq/install_hawq.sh @@ -126,7 +126,10 @@ cp -ar $UNZIP_DIR/docs $PREFIX/$DOC_DIR cp -ar $UNZIP_DIR/../libyarn/* $PREFIX/$LIB_DIR +sed -i -e 's#source %s/greenplum_path.sh" % hawq_home#source /etc/default/hawq#' $PREFIX/$BIN_DIR/hawq ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config +## HAWQ-422 configs are expected to sit in this hardcoded locations ;( +ln -s $PREFIX/$LIB_DIR/config $PREFIX/$LIB_DIR/etc wrapper=$PREFIX/usr/bin/hawq mkdir -p `dirname $wrapper` @@ -137,7 +140,7 @@ BIGTOP_DEFAULTS_DIR=\${BIGTOP_DEFAULTS_DIR-/etc/default} [ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hadoop ] && . \${BIGTOP_DEFAULTS_DIR}/hadoop [ -n "\${BIGTOP_DEFAULTS_DIR}" -a -r \${BIGTOP_DEFAULTS_DIR}/hawq ] && . \${BIGTOP_DEFAULTS_DIR}/hawq -exec /usr/lib/hawq/bin/hawq \$1 hawq +exec /usr/lib/hawq/bin/hawq \$@ EOF chmod 755 $wrapper http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5243a01/bigtop_toolchain/manifests/packages.pp ---------------------------------------------------------------------- diff --git a/bigtop_toolchain/manifests/packages.pp b/bigtop_toolchain/manifests/packages.pp index f378530..12a2918 100644 --- a/bigtop_toolchain/manifests/packages.pp +++ b/bigtop_toolchain/manifests/packages.pp @@ -22,16 +22,16 @@ class bigtop_toolchain::packages { } else { $mysql_devel="mariadb-devel" } - $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "snappy-devel" ] + $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "openssl-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite-devel", "openldap-devel", $mysql_devel, "rpm-build", "redhat-rpm-config", "fuse-libs", "asciidoc", "xmlto", "libyaml-devel", "gmp-devel", "readline-devel", "snappy-devel" ] } - /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "cppunit-devel", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "snappy-devel" ] + /(?i:(SLES|opensuse))/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "cppunit-devel", "gperf", "libopenssl-devel", "rpm-devel", "rpm-build", "pkg-config", "gmp-devel", "python-devel", "python-setuptools", "libxml2-devel", "libxslt-devel", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "cyrus-sasl-devel", "sqlite3-devel", "openldap2-devel", "libyaml-devel", "krb5-devel", "asciidoc", "xmlto", "libmysqlclient-devel", "readline-devel", "snappy-devel" ] # fix package dependencies: BIGTOP-2120 and BIGTOP-2152 exec { '/usr/bin/zypper remove -y krb5-mini': } -> exec {'/usr/bin/zypper install -y libopenssl-devel': } -> Package <| |> } - Amazon: { $pkgs = [ "unzip", "curl", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "fuse-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "snappy-devel" ] } - /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "libsnappy-dev" ] + Amazon: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "gcc-c++", "fuse", "createrepo", "lzo-devel", "bzip2-devel", "fuse-devel", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "openssl-devel", "rpm-build", "system-rpm-config", "fuse-libs","gmp-devel", "readline-devel", "snappy-devel" ] } + /(Ubuntu|Debian)/: { $pkgs = [ "unzip", "curl", "libcurl-devel", "wget", "git-core", "make", "cmake", "autoconf", "automake", "libtool", "apr-devel", "bison", "gcc", "g++", "fuse", "reprepro", "liblzo2-dev", "bzip2-devel", "libfuse-dev", "libcppunit-dev", "libssl-dev", "libzip-dev", "sharutils", "pkg-config", "debhelper", "devscripts", "build-essential", "dh-make", "libfuse2", "libssh-dev", "libjansi-java", "python2.7-dev", "libxml2-dev", "libxslt1-dev", "zlib1g-dev", "libsqlite3-dev", "libldap2-dev", "libsasl2-dev", "libmysqlclient-dev", "gperf", "libuuid-devel", "libgsasl-devel", "libevent-devel", "json-c-devel", "python-setuptools", "libkrb5-dev", "asciidoc", "libyaml-dev", "libgmp-dev", "readline-devel", "libsnappy-dev" ] exec { "apt-update": command => "/usr/bin/apt-get update"
