This is an automated email from the ASF dual-hosted git repository.
ayushtkn pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 61cc636fd TEZ-4727: Add ARM support of the dependent leveldbjni
(#509). (Ayush Saxena, reviewed by Laszlo Bodor)
61cc636fd is described below
commit 61cc636fdfffe871d3a52d9d739a668b0908995c
Author: Ayush Saxena <[email protected]>
AuthorDate: Tue Jun 23 17:39:02 2026 +0530
TEZ-4727: Add ARM support of the dependent leveldbjni (#509). (Ayush
Saxena, reviewed by Laszlo Bodor)
---
.github/workflows/build.yml | 5 +-
pom.xml | 21 +++++++-
tez-plugins/tez-aux-services/pom.xml | 32 +++++++++++-
.../org/apache/tez/auxservices/ShuffleHandler.java | 59 ++++++++++++++--------
4 files changed, 92 insertions(+), 25 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index befb80094..7c3870b86 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -36,7 +36,7 @@ jobs:
strategy:
matrix:
java-version: [21, 25]
- os: [ubuntu-latest, macos-latest]
+ os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v6
@@ -48,3 +48,6 @@ jobs:
run: >
mvn --batch-mode --no-transfer-progress clean install
-DskipTests -Dmaven.javadoc.skip=true
+ ${{ runner.os == 'Linux'
+ && runner.arch == 'ARM64'
+ && '-pl !tez-ui' || '' }}
diff --git a/pom.xml b/pom.xml
index bd1a268aa..0aadd69e3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -87,6 +87,8 @@
<jettison.version>1.5.4</jettison.version>
<junit.version>4.13.2</junit.version>
<junit.jupiter.version>5.9.3</junit.jupiter.version>
+ <leveldbjni.group>org.fusesource.leveldbjni</leveldbjni.group>
+ <leveldb.version>0.12</leveldb.version>
<leveldbjni-all.version>1.8</leveldbjni-all.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<metrics-core.version>3.1.0</metrics-core.version>
@@ -938,10 +940,15 @@
<version>${bouncycastle.version}</version>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
+ <groupId>${leveldbjni.group}</groupId>
<artifactId>leveldbjni-all</artifactId>
<version>${leveldbjni-all.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.iq80.leveldb</groupId>
+ <artifactId>leveldb</artifactId>
+ <version>${leveldb.version}</version>
+ </dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
@@ -1486,6 +1493,18 @@
</plugins>
</reporting>
</profile>
+ <profile>
+ <id>aarch64</id>
+ <properties>
+ <leveldbjni.group>org.openlabtesting.leveldbjni</leveldbjni.group>
+ </properties>
+ <activation>
+ <os>
+ <family>linux</family>
+ <arch>aarch64</arch>
+ </os>
+ </activation>
+ </profile>
</profiles>
<reporting>
diff --git a/tez-plugins/tez-aux-services/pom.xml
b/tez-plugins/tez-aux-services/pom.xml
index efa6fb107..dcebb15a5 100644
--- a/tez-plugins/tez-aux-services/pom.xml
+++ b/tez-plugins/tez-aux-services/pom.xml
@@ -49,6 +49,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -64,6 +70,10 @@
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -90,6 +100,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tez</groupId>
@@ -110,10 +126,14 @@
<scope>provided</scope>
</dependency>
<dependency>
- <groupId>org.fusesource.leveldbjni</groupId>
+ <groupId>${leveldbjni.group}</groupId>
<artifactId>leveldbjni-all</artifactId>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.iq80.leveldb</groupId>
+ <artifactId>leveldb</artifactId>
+ </dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
@@ -146,6 +166,10 @@
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -159,6 +183,12 @@
<artifactId>hadoop-yarn-server-tests</artifactId>
<scope>test</scope>
<type>test-jar</type>
+ <exclusions>
+ <exclusion>
+ <groupId>org.fusesource.leveldbjni</groupId>
+ <artifactId>leveldbjni-all</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git
a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
index 42f3c6834..60aa088be 100644
---
a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
+++
b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
@@ -29,8 +29,7 @@ import static
io.netty.handler.codec.http.HttpResponseStatus.NOT_FOUND;
import static io.netty.handler.codec.http.HttpResponseStatus.OK;
import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED;
import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-import static org.fusesource.leveldbjni.JniDBFactory.asString;
-import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.File;
import java.io.FileNotFoundException;
@@ -91,7 +90,6 @@ import
org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
-import org.apache.hadoop.yarn.server.utils.LeveldbIterator;
import org.apache.tez.common.security.JobTokenIdentifier;
import org.apache.tez.common.security.JobTokenSecretManager;
import org.apache.tez.runtime.library.common.Constants;
@@ -150,10 +148,10 @@ import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.CharsetUtil;
import io.netty.util.concurrent.GlobalEventExecutor;
-import org.fusesource.leveldbjni.JniDBFactory;
-import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
+import org.iq80.leveldb.DBFactory;
+import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
import org.slf4j.LoggerFactory;
@@ -211,6 +209,7 @@ public class ShuffleHandler extends AuxiliaryService {
private JobTokenSecretManager secretManager;
private DB stateDb = null;
+ private static final DBFactory DB_FACTORY = createDBFactory();
public static final String TEZ_SHUFFLE_SERVICEID =
"tez_shuffle";
@@ -644,6 +643,7 @@ public class ShuffleHandler extends AuxiliaryService {
destroyPipeline();
if (stateDb != null) {
stateDb.close();
+ stateDb = null;
}
super.serviceStop();
}
@@ -672,9 +672,9 @@ public class ShuffleHandler extends AuxiliaryService {
if (recoveryRoot != null) {
startStore(recoveryRoot);
Pattern jobPattern = Pattern.compile(JobID.JOBID_REGEX);
- LeveldbIterator iter = null;
+ DBIterator iter = null;
try {
- iter = new LeveldbIterator(stateDb);
+ iter = stateDb.iterator();
iter.seek(bytes(JobID.JOB));
while (iter.hasNext()) {
Map.Entry<byte[],byte[]> entry = iter.next();
@@ -696,26 +696,20 @@ public class ShuffleHandler extends AuxiliaryService {
private void startStore(Path recoveryRoot) throws IOException {
Options options = new Options();
- options.createIfMissing(false);
options.logger(new LevelDBLogger());
Path dbPath = new Path(recoveryRoot, STATE_DB_NAME);
LOG.info("Using state database at " + dbPath + " for recovery");
File dbfile = new File(dbPath.toString());
+ boolean dbExists = dbfile.exists();
+ options.createIfMissing(!dbExists);
try {
- stateDb = JniDBFactory.factory.open(dbfile, options);
- } catch (NativeDB.DBException e) {
- if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
- LOG.info("Creating state database at " + dbfile);
- options.createIfMissing(true);
- try {
- stateDb = JniDBFactory.factory.open(dbfile, options);
- storeVersion();
- } catch (DBException dbExc) {
- throw new IOException("Unable to create state store", dbExc);
- }
- } else {
- throw e;
- }
+ stateDb = DB_FACTORY.open(dbfile, options);
+ } catch (Exception e) {
+ throw new IOException("Unable to open state store", e);
+ }
+ if (!dbExists) {
+ LOG.info("Created state database at " + dbfile);
+ storeVersion();
}
checkVersion();
}
@@ -844,6 +838,27 @@ public class ShuffleHandler extends AuxiliaryService {
}
}
+ private static DBFactory createDBFactory() {
+ // Try native JNI LevelDB first (fastest), fall back to pure-Java iq80
+ try {
+ DBFactory jniFactory = org.fusesource.leveldbjni.JniDBFactory.factory;
+ LOG.info("Using JNI LevelDB factory");
+ return jniFactory;
+ } catch (Throwable t) {
+ LOG.info("Native LevelDB JNI library not available ({}), "
+ + "falling back to pure-Java implementation", t.getMessage());
+ return new org.iq80.leveldb.impl.Iq80DBFactory();
+ }
+ }
+
+ private static byte[] bytes(String value) {
+ return value.getBytes(UTF_8);
+ }
+
+ private static String asString(byte[] value) {
+ return new String(value, UTF_8);
+ }
+
private static class LevelDBLogger implements Logger {
private static final org.slf4j.Logger LOG =
LoggerFactory.getLogger(LevelDBLogger.class);
