This is an automated email from the ASF dual-hosted git repository.
guyuqi pushed a commit to branch openEuler-support
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/openEuler-support by this push:
new a33d98b36 BIGTOP-3967: add support for spark (#1148)
a33d98b36 is described below
commit a33d98b369a49f5095b7c436fedd741e4dd353d6
Author: MacChen01 <[email protected]>
AuthorDate: Tue Sep 12 10:07:52 2023 +0800
BIGTOP-3967: add support for spark (#1148)
---
bigtop-packages/src/rpm/spark/SPECS/spark.spec | 4 ++++
bigtop_toolchain/bin/puppetize.sh | 18 ++++++++++----
bigtop_toolchain/manifests/renv.pp | 33 +++++++++++++++++++++-----
provisioner/docker/config_openeuler-22.03.yaml | 2 +-
provisioner/utils/setup-env-centos.sh | 2 +-
5 files changed, 47 insertions(+), 12 deletions(-)
diff --git a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
index 3610a49f3..e213a117b 100644
--- a/bigtop-packages/src/rpm/spark/SPECS/spark.spec
+++ b/bigtop-packages/src/rpm/spark/SPECS/spark.spec
@@ -116,9 +116,13 @@ Server for Spark worker
%package -n %{spark_pkg_name}-python
Summary: Python client for Spark
Group: Development/Libraries
+%if 0%{?rhel} >= 8 || 0%{?openEuler}
%if 0%{?rhel} >= 8
Requires: %{spark_pkg_name}-core = %{version}-%{release}, python2
%else
+Requires: %{spark_pkg_name}-core = %{version}-%{release}, python3
+%endif
+%else
Requires: %{spark_pkg_name}-core = %{version}-%{release}, python
%endif
diff --git a/bigtop_toolchain/bin/puppetize.sh
b/bigtop_toolchain/bin/puppetize.sh
index 3fcf64ba4..b8c39c5a9 100755
--- a/bigtop_toolchain/bin/puppetize.sh
+++ b/bigtop_toolchain/bin/puppetize.sh
@@ -71,14 +71,24 @@ case ${ID}-${VERSION_ID} in
dnf config-manager --set-enabled codeready-builder-for-rhel-8-rhui-rpms
;;
openEuler-*)
- yum -y install hostname curl sudo unzip wget ruby vim patch
systemd-devel rng-tools findutils
+ dnf -y install hostname curl sudo unzip wget ruby ruby-devel vim
systemd-devel findutils 'dnf-command(config-manager)' nc initscripts
openeuler-lsb openssl-devel make gcc-c++ openEuler-rpm-config python3-pip
python3-devel dbus
+ dnf config-manager --add-repo
https://repo.oepkgs.net/openeuler/rpm/openEuler-22.03-LTS/extras/$HOSTTYPE
+ echo "gpgcheck=0" >>
/etc/yum.repos.d/repo.oepkgs.net_openeuler_rpm_openEuler-22.03-LTS_extras_$HOSTTYPE.repo
+ sed -i "s|enabled=1|enabled=1 \npriority=10|g"
/etc/yum.repos.d/openEuler.repo
+ dnf clean all
+ dnf makecache
# openEuler ruby version is 3.X,so use puppet-7.22.0.
- gem install puppet -v 7.22.0
- gem install xmlrpc
- gem install sync
+ gem install puppet:7.22.0 xmlrpc sync sys-filesystem
puppet module install puppetlabs-stdlib --version 4.12.0
#openEuler dnf defaulted is not use module,so comment module in
puppet-7.22.0
sed -i "91c execute([command(:dnf), 'install', '-d', '0', '-e',
self.class.error_level, '-y', args])"
/usr/local/share/gems/gems/puppet-7.22.0/lib/puppet/provider/package/dnfmodule.rb
+ #add python2 in openeuler
+ wget -P /usr/src/
https://www.python.org/ftp/python/2.7.14/Python-2.7.14.tgz
+ tar -xf /usr/src/Python-2.7.14.tgz -C /usr/src/
+ cd /usr/src/Python-2.7.14
+ ./configure --prefix=/usr/local/python2.7.14 --enable-optimizations
+ make && make install
+ ln -s /usr/local/python2.7.14/bin/python2.7 /usr/bin/python2
;;
*)
echo "Unsupported OS ${ID}-${VERSION_ID}."
diff --git a/bigtop_toolchain/manifests/renv.pp
b/bigtop_toolchain/manifests/renv.pp
index 484f4ace0..429a942ec 100644
--- a/bigtop_toolchain/manifests/renv.pp
+++ b/bigtop_toolchain/manifests/renv.pp
@@ -49,22 +49,43 @@ class bigtop_toolchain::renv {
]
}
}
+ /openEuler/: {
+ $pkgs = [
+ "R",
+ "R-devel",
+ ]
+ }
}
- if ($operatingsystem != 'openEuler') {
- package { $pkgs:
- ensure => installed,
- before => [Exec["install_r_packages"]]
+ #BIGTOP-3967: openEuler not support PowerPC currently.
+ if ($operatingsystem == 'openEuler'){
+ if ($architecture == "aarch64") {
+ $url =
"https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-arm64.tar.gz";
+ $pandoctar = "pandoc-2.19.2-linux-arm64.tar.gz"
+ } else{
+ $url =
"https://github.com/jgm/pandoc/releases/download/2.19.2/pandoc-2.19.2-linux-amd64.tar.gz";
+ $pandoctar = "pandoc-2.19.2-linux-amd64.tar.gz"
+ }
+
+ exec {"down_pandoc":
+ cwd => "/usr/src",
+ command => "/usr/bin/wget $url && /bin/tar -xvzf $pandoctar && ln -s
/usr/src/pandoc-2.19.2/bin/pandoc /usr/bin/pandoc",
}
}
+
+ package { $pkgs:
+ ensure => installed,
+ before => [Exec["install_r_packages"]]
+ }
+
+
# BIGTOP-3483:
# Upgrade R version to 3.6.3 to build Spark 3.0.1 on Ubuntu 16.04 and 18.04
#
# Then Install required R packages dependency
if (($operatingsystem == 'Ubuntu' and versioncmp($operatingsystemmajrelease,
'18.04') <= 0) or
- ($operatingsystem == 'Debian' and versioncmp($operatingsystemmajrelease,
'10') < 0) or
- ($operatingsystem == 'openEuler')) {
+ ($operatingsystem == 'Debian' and versioncmp($operatingsystemmajrelease,
'10') < 0)) {
$url = "https://cran.r-project.org/src/base/R-3/";
$rfile = "R-3.6.3.tar.gz"
$rdir = "R-3.6.3"
diff --git a/provisioner/docker/config_openeuler-22.03.yaml
b/provisioner/docker/config_openeuler-22.03.yaml
index 6745442b7..a8fbe6ed5 100644
--- a/provisioner/docker/config_openeuler-22.03.yaml
+++ b/provisioner/docker/config_openeuler-22.03.yaml
@@ -15,7 +15,7 @@
docker:
memory_limit: "4g"
- image: "bigtop/slaves:trunk-openeuler-22.03"
+ image: "bigtop/puppet:trunk-openeuler-22.03"
repo:
"http://repios.bigtop.apache.org/releases/3.2.0/openEuler/22.03/$basearch";
distro: centos
diff --git a/provisioner/utils/setup-env-centos.sh
b/provisioner/utils/setup-env-centos.sh
index 9822410e9..1b85eef76 100755
--- a/provisioner/utils/setup-env-centos.sh
+++ b/provisioner/utils/setup-env-centos.sh
@@ -31,7 +31,7 @@ fi
# BIGTOP-3883:
# yum-utils, yum-priorities and yum-config-manager are NOT available in
openEuler 22.03
if [ "${ID}" = "openEuler" ];then
- dnf install rng-tools 'dnf-command(config-manager)' -y
+ dnf install rng-tools -y
else
yum -y install rng-tools yum-priorities
fi
