This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 2870a5cca BIGTOP-4084. Upgrade Tez to 0.10.4. (#1246)
2870a5cca is described below
commit 2870a5cca8ed31c0f000ece9fec33c31fbe1a7fc
Author: Masatake Iwasaki <[email protected]>
AuthorDate: Sun Oct 27 20:23:00 2024 +0900
BIGTOP-4084. Upgrade Tez to 0.10.4. (#1246)
---
.../puppet/modules/tez/templates/environment | 1 +
bigtop-packages/src/common/hadoop/init-hcfs.groovy | 5 +++
bigtop-packages/src/common/tez/do-component-build | 10 -----
.../src/common/tez/patch2-TEZ-4319.diff | 50 ----------------------
.../src/common/tez/patch6-TEZ-4492.diff | 17 --------
.../src/common/tez/patch7-TEZ-4493.diff | 16 -------
.../src/common/tez/patch8-TEZ-4520.diff | 17 --------
bigtop-packages/src/common/tez/tez-site.xml | 4 +-
bigtop-tests/smoke-tests/tez/TestTezSmoke.groovy | 2 +-
bigtop.bom | 2 +-
10 files changed, 10 insertions(+), 114 deletions(-)
diff --git a/bigtop-deploy/puppet/modules/tez/templates/environment
b/bigtop-deploy/puppet/modules/tez/templates/environment
index 3b94c88fc..b38a910e7 100644
--- a/bigtop-deploy/puppet/modules/tez/templates/environment
+++ b/bigtop-deploy/puppet/modules/tez/templates/environment
@@ -15,3 +15,4 @@
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
HADOOP_CLASSPATH="/etc/tez/conf:/usr/lib/tez/*:/usr/lib/tez/lib/*"
+HADOOP_USER_CLASSPATH_FIRST="true"
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.groovy
b/bigtop-packages/src/common/hadoop/init-hcfs.groovy
index 398426a82..d633a7e8f 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.groovy
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.groovy
@@ -304,3 +304,8 @@ total_jars += copyJars(fs,
new Path(TEZ_APPS, "tez"))
LOG.info("Total jars copied into the DFS : " + total_jars);
+
+def tez_tar_gz = TEZ_HOME + "lib/tez.tar.gz"
+if (new File(tez_tar_gz).exists()) {
+ fs.copyFromLocalFile(new Path(tez_tar_gz), new Path(TEZ_APPS, "tez/lib"))
+}
diff --git a/bigtop-packages/src/common/tez/do-component-build
b/bigtop-packages/src/common/tez/do-component-build
index e72166976..2a5ca0c3d 100644
--- a/bigtop-packages/src/common/tez/do-component-build
+++ b/bigtop-packages/src/common/tez/do-component-build
@@ -18,16 +18,6 @@ set -xe
. `dirname $0`/bigtop.bom
-if [ $HOSTTYPE = "powerpc64le" ] ; then
- mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc
-Dversion=2.5.0 \
- -Dclassifier=linux-ppcle_64 -Dpackaging=exe
-Dfile=/usr/local/protobuf-2.5.0/bin/protoc
-fi
-
-if [ $HOSTTYPE = "aarch64" ] ; then
- mvn install:install-file -DgroupId=com.google.protobuf -DartifactId=protoc
-Dversion=2.5.0 \
- -Dclassifier=linux-aarch_64 -Dpackaging=exe
-Dfile=/usr/local/protobuf-2.5.0/bin/protoc
-fi
-
BUILD_TEZ_OPTS="clean package \
-Dtar -Dhadoop.version=${HADOOP_VERSION} \
-Phadoop28 \
diff --git a/bigtop-packages/src/common/tez/patch2-TEZ-4319.diff
b/bigtop-packages/src/common/tez/patch2-TEZ-4319.diff
deleted file mode 100644
index 40e9e687d..000000000
--- a/bigtop-packages/src/common/tez/patch2-TEZ-4319.diff
+++ /dev/null
@@ -1,50 +0,0 @@
-diff --git a/pom.xml b/pom.xml
-index 840ab74e4..893a68fa9 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -65,6 +65,7 @@
- <slf4j.version>1.7.36</slf4j.version>
- <protobuf.version>2.5.0</protobuf.version>
- <roaringbitmap.version>0.7.45</roaringbitmap.version>
-+ <snappy-java.version>1.1.8.4</snappy-java.version>
- <protoc.path>${env.PROTOC_PATH}</protoc.path>
- <scm.url>scm:git:https://gitbox.apache.org/repos/asf/tez.git</scm.url>
- <frontend-maven-plugin.version>1.8.0</frontend-maven-plugin.version>
-@@ -363,6 +364,10 @@
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
-+ <exclusion>
-+ <groupId>org.xerial.snappy</groupId>
-+ <artifactId>snappy-java</artifactId>
-+ </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
-@@ -764,6 +769,11 @@
- <artifactId>leveldbjni-all</artifactId>
- <version>1.8</version>
- </dependency>
-+ <dependency>
-+ <groupId>org.xerial.snappy</groupId>
-+ <artifactId>snappy-java</artifactId>
-+ <version>${snappy-java.version}</version>
-+ </dependency>
- </dependencies>
- </dependencyManagement>
-
-diff --git a/tez-api/pom.xml b/tez-api/pom.xml
-index 5c11fdc3b..57acc4e12 100644
---- a/tez-api/pom.xml
-+++ b/tez-api/pom.xml
-@@ -115,6 +115,10 @@
- <artifactId>bcprov-jdk15on</artifactId>
- <scope>test</scope>
- </dependency>
-+ <dependency>
-+ <groupId>org.xerial.snappy</groupId>
-+ <artifactId>snappy-java</artifactId>
-+ </dependency>
- </dependencies>
-
- <build>
diff --git a/bigtop-packages/src/common/tez/patch6-TEZ-4492.diff
b/bigtop-packages/src/common/tez/patch6-TEZ-4492.diff
deleted file mode 100644
index 8797059df..000000000
--- a/bigtop-packages/src/common/tez/patch6-TEZ-4492.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-commit ccad71dabceb850b58ff38e77fd206e68f62ce4b
-Author: AnmolSun <[email protected]>
-Date: Thu May 4 13:26:14 2023 +0530
-
- TEZ-4492: Update Bowerrc to use bower.herokuapp mirror to avoid Bower
Registry CERT_EXPIRE issue (BOWER-2608) (#284) (Anmol Sundaram reviewed by
Laszlo Bodor)
-
-diff --git a/tez-ui/src/main/webapp/.bowerrc b/tez-ui/src/main/webapp/.bowerrc
-index 5b0b07d75..b798d4977 100644
---- a/tez-ui/src/main/webapp/.bowerrc
-+++ b/tez-ui/src/main/webapp/.bowerrc
-@@ -1,5 +1,6 @@
- {
- "directory": "bower_components",
-+ "registry": "https://bower.herokuapp.com";,
- "analytics": false,
- "resolvers": [
- "bower-shrinkwrap-resolver-ext"
diff --git a/bigtop-packages/src/common/tez/patch7-TEZ-4493.diff
b/bigtop-packages/src/common/tez/patch7-TEZ-4493.diff
deleted file mode 100644
index 59f861519..000000000
--- a/bigtop-packages/src/common/tez/patch7-TEZ-4493.diff
+++ /dev/null
@@ -1,16 +0,0 @@
-diff --git a/tez-dag/src/main/java/org/apache/tez/state/StateMachineTez.java
b/tez-dag/src/main/java/org/apache/tez/state/StateMachineTez.java
-index 3be771892..cbb838d7f 100644
---- a/tez-dag/src/main/java/org/apache/tez/state/StateMachineTez.java
-+++ b/tez-dag/src/main/java/org/apache/tez/state/StateMachineTez.java
-@@ -51,6 +51,11 @@ public class StateMachineTez<STATE extends Enum<STATE>,
EVENTTYPE extends Enum<E
- return realStatemachine.getCurrentState();
- }
-
-+ @Override
-+ public STATE getPreviousState() {
-+ return realStatemachine.getPreviousState();
-+ }
-+
- @SuppressWarnings("unchecked")
- @Override
- public STATE doTransition(EVENTTYPE eventType, EVENT event) throws
diff --git a/bigtop-packages/src/common/tez/patch8-TEZ-4520.diff
b/bigtop-packages/src/common/tez/patch8-TEZ-4520.diff
deleted file mode 100644
index 14852a7de..000000000
--- a/bigtop-packages/src/common/tez/patch8-TEZ-4520.diff
+++ /dev/null
@@ -1,17 +0,0 @@
-diff --git a/tez-dist/pom.xml b/tez-dist/pom.xml
-index 8b7f445ef..14ae62f27 100644
---- a/tez-dist/pom.xml
-+++ b/tez-dist/pom.xml
-@@ -49,6 +49,12 @@
- <version>${project.version}</version>
- <type>test-jar</type>
- </dependency>
-+ <dependency>
-+ <groupId>org.apache.tez</groupId>
-+ <artifactId>tez-job-analyzer</artifactId>
-+ <version>${project.version}</version>
-+ <scope>provided</scope>
-+ </dependency>
- </dependencies>
-
- <properties>
diff --git a/bigtop-packages/src/common/tez/tez-site.xml
b/bigtop-packages/src/common/tez/tez-site.xml
index cd29d2669..48e51a180 100644
--- a/bigtop-packages/src/common/tez/tez-site.xml
+++ b/bigtop-packages/src/common/tez/tez-site.xml
@@ -17,11 +17,11 @@
<configuration>
<property>
<name>tez.lib.uris</name>
-<value>${fs.default.name}/apps/tez/,${fs.default.name}/apps/tez/lib/</value>
+<value>${fs.default.name}/apps/tez/lib/tez.tar.gz</value>
</property>
<property>
<name>tez.use.cluster.hadoop-libs</name>
-<value>true</value>
+<value>false</value>
</property>
</configuration>
diff --git a/bigtop-tests/smoke-tests/tez/TestTezSmoke.groovy
b/bigtop-tests/smoke-tests/tez/TestTezSmoke.groovy
index a1aa4daf2..a1c3b5455 100644
--- a/bigtop-tests/smoke-tests/tez/TestTezSmoke.groovy
+++ b/bigtop-tests/smoke-tests/tez/TestTezSmoke.groovy
@@ -33,7 +33,7 @@ class TestTezSmoke {
static final String HADDOP_CMD = "hadoop jar "
static final String TEZ_EXMP_JAR = TEZ_HOME + "/tez-examples-*.jar "
static final String TEZ_TEST_JAR = TEZ_HOME + "/tez-tests-*.jar "
- static final String SET_HADOOP_CLASSPATH = "export
HADOOP_CLASSPATH=/etc/tez/conf:/usr/lib/tez/*:/usr/lib/tez/lib/*;"
+ static final String SET_HADOOP_CLASSPATH = "export
HADOOP_CLASSPATH=/etc/tez/conf:/usr/lib/tez/*:/usr/lib/tez/lib/*; export
HADOOP_USER_CLASSPATH_FIRST=true; "
@BeforeClass
static void TezSetUp() {
diff --git a/bigtop.bom b/bigtop.bom
index 8e6db945d..097519e31 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -193,7 +193,7 @@ bigtop {
name = 'tez'
rpm_pkg_suffix = "_" + bigtop.base_version.replace(".", "_")
relNotes = 'Apache TEZ'
- version { base = '0.10.2'; pkg = base; release = 1 }
+ version { base = '0.10.4'; pkg = base; release = 1 }
tarball { destination = "apache-${name}-${version.base}-src.tar.gz"
source = destination }
url { download_path = "/$name/${version.base}/"
