This is an automated email from the ASF dual-hosted git repository. guyuqi pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/bigtop.git
commit 055d82562db1dfd2026be307a6b30a774e6b649d Author: MacChen01 <[email protected]> AuthorDate: Tue Sep 12 10:07:52 2023 +0800 BIGTOP-3967: add support for spark (#1148) (cherry picked from commit a33d98b369a49f5095b7c436fedd741e4dd353d6) --- bigtop-packages/src/rpm/spark/SPECS/spark.spec | 4 ++++ bigtop_toolchain/bin/puppetize.sh | 18 ++++++++++---- bigtop_toolchain/manifests/renv.pp | 33 +++++++++++++++++++++----- provisioner/docker/config_openeuler-22.03.yaml | 2 +- provisioner/utils/setup-env-centos.sh | 2 +- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec b/bigtop-packages/src/rpm/spark/SPECS/spark.spec index 3610a49f3..e213a117b 100644 --- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec +++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec @@ -116,9 +116,13 @@ Server for Spark worker %package -n %{spark_pkg_name}-python Summary: Python client for Spark Group: Development/Libraries +%if 0%{?rhel} >= 8 || 0%{?openEuler} %if 0%{?rhel} >= 8 Requires: %{spark_pkg_name}-core = %{version}-%{release}, python2 %else +Requires: %{spark_pkg_name}-core = %{version}-%{release}, python3 +%endif +%else Requires: %{spark_pkg_name}-core = %{version}-%{release}, python %endif diff --git a/bigtop_toolchain/bin/puppetize.sh b/bigtop_toolchain/bin/puppetize.sh index 3fcf64ba4..b8c39c5a9 100755 --- a/bigtop_toolchain/bin/puppetize.sh +++ b/bigtop_toolchain/bin/puppetize.sh @@ -71,14 +71,24 @@ case ${ID}-${VERSION_ID} in dnf config-manager --set-enabled codeready-builder-for-rhel-8-rhui-rpms ;; openEuler-*) - yum -y install hostname curl sudo unzip wget ruby vim patch systemd-devel rng-tools findutils + dnf -y install hostname curl sudo unzip wget ruby ruby-devel vim systemd-devel findutils 'dnf-command(config-manager)' nc initscripts openeuler-lsb openssl-devel make gcc-c++ openEuler-rpm-config python3-pip python3-devel dbus + dnf config-manager --add-repo https://repo.oepkgs.net/openeuler/rpm/openEuler-22.03-LTS/extras/$HOSTTYPE + echo "gpgcheck=0" >> /etc/yum.repos.d/repo.oepkgs.net_openeuler_rpm_openEuler-22.03-LTS_extras_$HOSTTYPE.repo + sed -i "s|enabled=1|enabled=1 \npriority=10|g" /etc/yum.repos.d/openEuler.repo + dnf clean all + dnf makecache # openEuler ruby version is 3.X,so use puppet-7.22.0. - gem install puppet -v 7.22.0 - gem install xmlrpc - gem install sync + gem install puppet:7.22.0 xmlrpc sync sys-filesystem puppet module install puppetlabs-stdlib --version 4.12.0 #openEuler dnf defaulted is not use module,so comment module in puppet-7.22.0 sed -i "91c execute([command(:dnf), 'install', '-d', '0', '-e', self.class.error_level, '-y', args])" /usr/local/share/gems/gems/puppet-7.22.0/lib/puppet/provider/package/dnfmodule.rb + #add python2 in openeuler + wget -P /usr/src/ https://www.python.org/ftp/python/2.7.14/Python-2.7.14.tgz + tar -xf /usr/src/Python-2.7.14.tgz -C /usr/src/ + cd /usr/src/Python-2.7.14 + ./configure --prefix=/usr/local/python2.7.14 --enable-optimizations + make && make install + ln -s /usr/local/python2.7.14/bin/python2.7 /usr/bin/python2 ;; *) echo "Unsupported OS ${ID}-${VERSION_ID}." diff --git a/bigtop_toolchain/manifests/renv.pp b/bigtop_toolchain/manifests/renv.pp index 484f4ace0..429a942ec 100644 --- a/bigtop_toolchain/manifests/renv.pp +++ b/bigtop_toolchain/manifests/renv.pp @@ -49,22 +49,43 @@ class bigtop_toolchain::renv { ] } } + /openEuler/: { + $pkgs = [ + "R", + "R-devel", + ] + } } - if ($operatingsystem != 'openEuler') { - package { $pkgs: - ensure => installed, - before => [Exec["install_r_packages"]] + #BIGTOP-3967: openEuler not support PowerPC currently. + if ($operatingsystem == 'openEuler'){ + if ($architecture == "aarch64") { + $url = "https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-arm64.tar.gz"; + $pandoctar = "pandoc-2.19.2-linux-arm64.tar.gz" + } else{ + $url = "https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-amd64.tar.gz"; + $pandoctar = "pandoc-2.19.2-linux-amd64.tar.gz" + } + + exec {"down_pandoc": + cwd => "/usr/src", + command => "/usr/bin/wget $url && /bin/tar -xvzf $pandoctar && ln -s /usr/src/pandoc-2.19.2/bin/pandoc /usr/bin/pandoc", } } + + package { $pkgs: + ensure => installed, + before => [Exec["install_r_packages"]] + } + + # BIGTOP-3483: # Upgrade R version to 3.6.3 to build Spark 3.0.1 on Ubuntu 16.04 and 18.04 # # Then Install required R packages dependency if (($operatingsystem == 'Ubuntu' and versioncmp($operatingsystemmajrelease, '18.04') <= 0) or - ($operatingsystem == 'Debian' and versioncmp($operatingsystemmajrelease, '10') < 0) or - ($operatingsystem == 'openEuler')) { + ($operatingsystem == 'Debian' and versioncmp($operatingsystemmajrelease, '10') < 0)) { $url = "https://cran.r-project.org/src/base/R-3/"; $rfile = "R-3.6.3.tar.gz" $rdir = "R-3.6.3" diff --git a/provisioner/docker/config_openeuler-22.03.yaml b/provisioner/docker/config_openeuler-22.03.yaml index 6745442b7..a8fbe6ed5 100644 --- a/provisioner/docker/config_openeuler-22.03.yaml +++ b/provisioner/docker/config_openeuler-22.03.yaml @@ -15,7 +15,7 @@ docker: memory_limit: "4g" - image: "bigtop/slaves:trunk-openeuler-22.03" + image: "bigtop/puppet:trunk-openeuler-22.03" repo: "http://repios.bigtop.apache.org/releases/3.2.0/openEuler/22.03/$basearch"; distro: centos diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh index 9822410e9..1b85eef76 100755 --- a/provisioner/utils/setup-env-centos.sh +++ b/provisioner/utils/setup-env-centos.sh @@ -31,7 +31,7 @@ fi # BIGTOP-3883: # yum-utils, yum-priorities and yum-config-manager are NOT available in openEuler 22.03 if [ "${ID}" = "openEuler" ];then - dnf install rng-tools 'dnf-command(config-manager)' -y + dnf install rng-tools -y else yum -y install rng-tools yum-priorities fi
