This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 8db007a BIGTOP-3484. Bump Alluxio to 2.4.1. (#724)
8db007a is described below
commit 8db007af74b2f7a44a2d8ed2909f08ca15331a36
Author: Kengo Seki <[email protected]>
AuthorDate: Fri Feb 12 17:05:06 2021 +0900
BIGTOP-3484. Bump Alluxio to 2.4.1. (#724)
---
bigtop-deploy/puppet/modules/alluxio/manifests/init.pp | 14 ++++++++++++++
.../{alluxio-worker.svc => alluxio-job-master.svc} | 16 ++++++----------
.../{alluxio-worker.svc => alluxio-job-worker.svc} | 14 +++++++-------
bigtop-packages/src/common/alluxio/alluxio-worker.svc | 2 +-
bigtop-packages/src/common/alluxio/do-component-build | 6 +-----
bigtop-packages/src/common/alluxio/install_alluxio.sh | 5 -----
.../src/common/alluxio/patch0-fix-license-check.diff | 6 +++---
.../alluxio/patch1-fix-hadoop-version-check.diff | 13 -------------
.../src/common/alluxio/patch2-fix-jnr-fuse-repo.diff | 18 ------------------
bigtop-packages/src/deb/alluxio/rules | 2 ++
bigtop-packages/src/rpm/alluxio/SPECS/alluxio.spec | 6 +++++-
.../smoke-tests/alluxio/TestAlluxioSmoke.groovy | 5 +++--
bigtop.bom | 2 +-
13 files changed, 43 insertions(+), 66 deletions(-)
diff --git a/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
b/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
index 9200e92..7d44917 100644
--- a/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/alluxio/manifests/init.pp
@@ -57,6 +57,13 @@ class alluxio {
hasrestart => true,
hasstatus => true,
}
+ service { "alluxio-job-master":
+ ensure => running,
+ require => [ Package["alluxio"], Exec["daemon-reload"], Exec["alluxio
formatting"] ],
+ subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
+ hasrestart => true,
+ hasstatus => true,
+ }
}
}
@@ -77,5 +84,12 @@ class alluxio {
hasrestart => true,
hasstatus => true,
}
+ service { "alluxio-job-worker":
+ ensure => running,
+ require => [ Package["alluxio"], Exec["daemon-reload"],
File["/etc/alluxio/conf/alluxio-site.properties"] ],
+ subscribe => File["/etc/alluxio/conf/alluxio-site.properties"],
+ hasrestart => true,
+ hasstatus => true,
+ }
}
}
diff --git a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
b/bigtop-packages/src/common/alluxio/alluxio-job-master.svc
similarity index 85%
copy from bigtop-packages/src/common/alluxio/alluxio-worker.svc
copy to bigtop-packages/src/common/alluxio/alluxio-job-master.svc
index 2e69658..a9ac0d0 100644
--- a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
+++ b/bigtop-packages/src/common/alluxio/alluxio-job-master.svc
@@ -13,13 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-TYPE="worker"
+TYPE="job-master"
DAEMON="alluxio-${TYPE}"
DESC="Alluxio ${TYPE}"
-EXEC_PATH="/usr/lib/alluxio/bin/alluxio-start.sh"
+EXEC_PATH="/usr/lib/alluxio/bin"
WORKING_DIR="/var/lib/alluxio"
-DAEMON_FLAGS="worker Mount"
-KEYWORD="alluxio.worker.AlluxioWorker"
+DAEMON_FLAGS="job_master"
+KEYWORD="alluxio.master.AlluxioJobMaster"
CONF_DIR="/etc/alluxio/conf"
PIDFILE="/var/run/alluxio/${DAEMON}.pid"
@@ -39,12 +39,8 @@ start() {
LOG_FILE=/var/log/alluxio/${DAEMON}.out
- if [ -f $CONF_DIR/alluxio-env.sh ]; then
- . $CONF_DIR/alluxio-env.sh
- fi
-
/bin/bash -c "nohup nice -n 0 \
- ${EXEC_PATH} ${DAEMON_FLAGS} \
+ ${EXEC_PATH}/alluxio-start.sh ${DAEMON_FLAGS} \
> $LOG_FILE 2>&1 & "
WAIT_TIMES=0
@@ -68,7 +64,7 @@ generate_stop() {
cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
- /usr/lib/alluxio/bin/alluxio-stop.sh worker 2>&1 > /dev/null
+ ${EXEC_PATH}/alluxio-stop.sh ${DAEMON_FLAGS} 2>&1 > /dev/null
RETVAL=$?
[ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
diff --git a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
b/bigtop-packages/src/common/alluxio/alluxio-job-worker.svc
similarity index 87%
copy from bigtop-packages/src/common/alluxio/alluxio-worker.svc
copy to bigtop-packages/src/common/alluxio/alluxio-job-worker.svc
index 2e69658..a29da4c 100644
--- a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
+++ b/bigtop-packages/src/common/alluxio/alluxio-job-worker.svc
@@ -13,13 +13,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-TYPE="worker"
+TYPE="job-worker"
DAEMON="alluxio-${TYPE}"
DESC="Alluxio ${TYPE}"
-EXEC_PATH="/usr/lib/alluxio/bin/alluxio-start.sh"
+EXEC_PATH="/usr/lib/alluxio/bin"
WORKING_DIR="/var/lib/alluxio"
-DAEMON_FLAGS="worker Mount"
-KEYWORD="alluxio.worker.AlluxioWorker"
+DAEMON_FLAGS="job_worker"
+KEYWORD="alluxio.worker.AlluxioJobWorker"
CONF_DIR="/etc/alluxio/conf"
PIDFILE="/var/run/alluxio/${DAEMON}.pid"
@@ -44,11 +44,11 @@ start() {
fi
/bin/bash -c "nohup nice -n 0 \
- ${EXEC_PATH} ${DAEMON_FLAGS} \
+ ${EXEC_PATH}/alluxio-start.sh ${DAEMON_FLAGS} \
> $LOG_FILE 2>&1 & "
WAIT_TIMES=0
- until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [
$WAIT_TIMES -eq 10 ]; do
+ until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [
$WAIT_TIMES -eq 30 ]; do
(( WAIT_TIMES++ ))
sleep 1
done
@@ -68,7 +68,7 @@ generate_stop() {
cat <<'__EOT__'
stop() {
log_success_msg "Stopping $DESC (${DAEMON}): "
- /usr/lib/alluxio/bin/alluxio-stop.sh worker 2>&1 > /dev/null
+ ${EXEC_PATH}/alluxio-stop.sh ${DAEMON_FLAGS} 2>&1 > /dev/null
RETVAL=$?
[ $RETVAL -eq $RETVAL_SUCCESS ] && rm -f $LOCKFILE $PIDFILE
diff --git a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
b/bigtop-packages/src/common/alluxio/alluxio-worker.svc
index 2e69658..24e84f0 100644
--- a/bigtop-packages/src/common/alluxio/alluxio-worker.svc
+++ b/bigtop-packages/src/common/alluxio/alluxio-worker.svc
@@ -48,7 +48,7 @@ start() {
> $LOG_FILE 2>&1 & "
WAIT_TIMES=0
- until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [
$WAIT_TIMES -eq 10 ]; do
+ until ps -A -o pid,command | grep -i "[j]ava" | grep $KEYWORD || [
$WAIT_TIMES -eq 30 ]; do
(( WAIT_TIMES++ ))
sleep 1
done
diff --git a/bigtop-packages/src/common/alluxio/do-component-build
b/bigtop-packages/src/common/alluxio/do-component-build
index 961e62a..f6ba380 100644
--- a/bigtop-packages/src/common/alluxio/do-component-build
+++ b/bigtop-packages/src/common/alluxio/do-component-build
@@ -18,8 +18,4 @@ set -ex
. `dirname $0`/bigtop.bom
-t="hadoop.version"
-sed -i "s#<$t>.*</$t>#<$t>${HADOOP_VERSION}</$t>#" pom.xml
-sed -i "s|http://repo.spring.io|https://repo.spring.io|g" pom.xml
-
-mvn clean install -DskipTests -Pspark -Pyarn "$@"
+mvn clean install -DskipTests -Dhadoop.version=3.2.2 -Phadoop-3 -Pyarn "$@"
diff --git a/bigtop-packages/src/common/alluxio/install_alluxio.sh
b/bigtop-packages/src/common/alluxio/install_alluxio.sh
index 23f3f0a..015febc 100644
--- a/bigtop-packages/src/common/alluxio/install_alluxio.sh
+++ b/bigtop-packages/src/common/alluxio/install_alluxio.sh
@@ -124,14 +124,12 @@ cp -a lib/* $PREFIX/$LIB_DIR/lib
cp -a libexec/* $PREFIX/$LIB_DIR/libexec
cp -a client/* $PREFIX/$LIB_DIR/client
cp -a integration/* $PREFIX/$LIB_DIR/integration
-cp integration/checker/target/alluxio-checker-*-jar-with-dependencies.jar
$PREFIX/$LIB_DIR/integration/checker
cp
integration/fuse/target/alluxio-integration-fuse-*-jar-with-dependencies.jar
$PREFIX/$LIB_DIR/integration/fuse
cp
integration/yarn/target/alluxio-integration-yarn-*-jar-with-dependencies.jar
$PREFIX/$LIB_DIR/integration/yarn
rm -rf $PREFIX/$LIB_DIR/integration/pom.xml
$PREFIX/$LIB_DIR/integration/**/pom.xml
rm -rf $PREFIX/$LIB_DIR/integration/target
$PREFIX/$LIB_DIR/integration/**/target
rm -rf $PREFIX/$LIB_DIR/integration/**/src
rm -rf $PREFIX/$LIB_DIR/integration/**/README.md
-cp -rf core/server/common/src/main/webapp
$PREFIX/$LIB_DIR/core/server/common/src/main
# Copy in the configuration files
install -m 0644 conf/log4j.properties $PREFIX/$CONF_DIST_DIR
@@ -145,9 +143,6 @@ cp conf/workers $PREFIX/$CONF_DIST_DIR/workers
# Copy in the /usr/bin/alluxio wrapper
install -d -m 0755 $PREFIX/$BIN_DIR
-# Copy in alluxio deploy scripts
-cp -rf deploy $PREFIX/$LIB_DIR/share
-
# Prefix is correct at time of install,
# but we dont want to escape it before that point.
cat > $PREFIX/$BIN_DIR/alluxio <<EOF
diff --git a/bigtop-packages/src/common/alluxio/patch0-fix-license-check.diff
b/bigtop-packages/src/common/alluxio/patch0-fix-license-check.diff
index 75da34b..9f6c393 100644
--- a/bigtop-packages/src/common/alluxio/patch0-fix-license-check.diff
+++ b/bigtop-packages/src/common/alluxio/patch0-fix-license-check.diff
@@ -1,10 +1,10 @@
diff --git a/pom.xml b/pom.xml
-index a959527aa2..1051f153e2 100644
+index 8b6440b7e7..4ecbca4be4 100644
--- a/pom.xml
+++ b/pom.xml
-@@ -1175,6 +1175,7 @@
-
+@@ -1214,6 +1214,7 @@
<!-- Build and Packaging Exclusions -->
+ <exclude>webui/node_modules/*</exclude>
<exclude>build/**/*</exclude>
+ <exclude>debian/**/*</exclude>
<exclude>conf/alluxio-env.sh</exclude>
diff --git
a/bigtop-packages/src/common/alluxio/patch1-fix-hadoop-version-check.diff
b/bigtop-packages/src/common/alluxio/patch1-fix-hadoop-version-check.diff
deleted file mode 100644
index 29c4dba..0000000
--- a/bigtop-packages/src/common/alluxio/patch1-fix-hadoop-version-check.diff
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git a/integration/yarn/pom.xml b/integration/yarn/pom.xml
-index 7196350..e11713c 100644
---- a/integration/yarn/pom.xml
-+++ b/integration/yarn/pom.xml
-@@ -90,7 +90,7 @@
- <rules>
- <requireProperty>
- <property>hadoop.version</property>
-- <regex>[^12]\..*|2\.[4-9]\..*</regex>
-+ <regex>[^12]\..*|2\.([4-9]|[1-9]\d{1,})\..*</regex>
- <regexMessage>
- hadoop.version must be 2.4.0 or later to build yarn
integration.
- </regexMessage>
diff --git a/bigtop-packages/src/common/alluxio/patch2-fix-jnr-fuse-repo.diff
b/bigtop-packages/src/common/alluxio/patch2-fix-jnr-fuse-repo.diff
deleted file mode 100644
index 4526e69..0000000
--- a/bigtop-packages/src/common/alluxio/patch2-fix-jnr-fuse-repo.diff
+++ /dev/null
@@ -1,18 +0,0 @@
-diff --git a/pom.xml b/pom.xml
-index fc9e13a149..162824764f 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -100,10 +100,9 @@
- </snapshots>
- </repository>
- <repository>
-- <!-- for Pivotal's distribution -->
-- <id>spring-releases</id>
-- <name>Spring Release Repository</name>
-- <url>http://repo.spring.io/libs-release</url>
-+ <id>bintray</id>
-+ <name>bintray</name>
-+ <url>https://jcenter.bintray.com</url>
- <releases>
- <enabled>true</enabled>
- </releases>
diff --git a/bigtop-packages/src/deb/alluxio/rules
b/bigtop-packages/src/deb/alluxio/rules
index bac4f9f..6ab93d5 100755
--- a/bigtop-packages/src/deb/alluxio/rules
+++ b/bigtop-packages/src/deb/alluxio/rules
@@ -41,3 +41,5 @@ override_dh_auto_install:
mkdir -p debian/alluxio/etc/init.d/
bash debian/init.d.tmpl debian/alluxio-master.svc deb
debian/alluxio/etc/init.d/alluxio-master
bash debian/init.d.tmpl debian/alluxio-worker.svc deb
debian/alluxio/etc/init.d/alluxio-worker
+ bash debian/init.d.tmpl debian/alluxio-job-master.svc deb
debian/alluxio/etc/init.d/alluxio-job-master
+ bash debian/init.d.tmpl debian/alluxio-job-worker.svc deb
debian/alluxio/etc/init.d/alluxio-job-worker
diff --git a/bigtop-packages/src/rpm/alluxio/SPECS/alluxio.spec
b/bigtop-packages/src/rpm/alluxio/SPECS/alluxio.spec
index c506069..dd6fa18 100644
--- a/bigtop-packages/src/rpm/alluxio/SPECS/alluxio.spec
+++ b/bigtop-packages/src/rpm/alluxio/SPECS/alluxio.spec
@@ -29,9 +29,11 @@ Source2: install_alluxio.sh
Source3: init.d.tmpl
Source4: alluxio-master.svc
Source5: alluxio-worker.svc
+Source6: alluxio-job-master.svc
+Source7: alluxio-job-worker.svc
#BIGTOP_PATCH_FILES
%define alluxio_home /usr/lib/%{alluxio_name}
-%define alluxio_services master worker
+%define alluxio_services master worker job-master job-worker
%define var_lib /var/lib/%{alluxio_name}
%define var_run /var/run/%{alluxio_name}
%define var_log /var/log/%{alluxio_name}
@@ -136,6 +138,8 @@ done
%config(noreplace) %{_sysconfdir}/%{alluxio_name}/conf.dist/core-site.xml
%config(noreplace) %{initd_dir}/%{alluxio_name}-master
%config(noreplace) %{initd_dir}/%{alluxio_name}-worker
+%config(noreplace) %{initd_dir}/%{alluxio_name}-job-master
+%config(noreplace) %{initd_dir}/%{alluxio_name}-job-worker
%attr(0755,alluxio,alluxio) %{var_lib}
%attr(0755,alluxio,alluxio) %{var_run}
%attr(0755,alluxio,alluxio) %{var_log}
diff --git a/bigtop-tests/smoke-tests/alluxio/TestAlluxioSmoke.groovy
b/bigtop-tests/smoke-tests/alluxio/TestAlluxioSmoke.groovy
index 60c56c2..5b1ff32 100644
--- a/bigtop-tests/smoke-tests/alluxio/TestAlluxioSmoke.groovy
+++ b/bigtop-tests/smoke-tests/alluxio/TestAlluxioSmoke.groovy
@@ -66,11 +66,12 @@ class TestAlluxioSmoke {
sh.exec("""
set -x
# Clear test environment
- $hadoop fs -rm $alluxioTestDir/hadoopLs/datafile
- $alluxio fs rm $alluxioTestDir/hadoopLs/datafile
+ $hadoop fs -rm -r /underFSStorage/$alluxioTestDir
+ $alluxio fs rm -R $alluxioTestDir
set -e
# Test Alluxio and HDFS interoperability
+ $alluxio fs mkdir $alluxioTestDir/hadoopLs
$alluxio fs copyFromLocal datafile $alluxioTestDir/hadoopLs/datafile
$alluxio fs persist $alluxioTestDir/hadoopLs/datafile
$hadoop fs -cat /underFSStorage/$alluxioTestDir/hadoopLs/datafile
diff --git a/bigtop.bom b/bigtop.bom
index cae2f50..e25eb0a 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -352,7 +352,7 @@ bigtop {
name = "alluxio"
pkg = "alluxio"
relNotes = "Alluxio: a memory-centric distributed file system"
- version { base = '1.8.2'; pkg = base; release = 1 }
+ version { base = '2.4.1'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "v${version.base}.tar.gz" }
url { site = "https://github.com/Alluxio/alluxio/archive";
