This is an automated email from the ASF dual-hosted git repository.
richox pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new 3eb0e735 [AURON #1509] Deprecated JniBridge and AuronCallNativeWrapper
under the spark-extension module (#1510)
3eb0e735 is described below
commit 3eb0e7353d0e280ddc7ae3dab77b8206c31dfb76
Author: zhangmang <[email protected]>
AuthorDate: Mon Oct 27 10:40:20 2025 +0800
[AURON #1509] Deprecated JniBridge and AuronCallNativeWrapper under the
spark-extension module (#1510)
* [AURON #1509] Deprecated JniBridge and AuronCallNativeWrapper under the
spark-extension module
* fix checkstyle
* fix checkstyle
---
.../apache/auron/jni/AuronCallNativeWrapper.java | 5 +++++
.../java/org/apache/spark/sql/auron/JniBridge.java | 25 ++++++++++++++++++++++
.../spark/sql/auron/AuronCallNativeWrapper.scala | 13 +++++++++++
3 files changed, 43 insertions(+)
diff --git
a/auron-core/src/main/java/org/apache/auron/jni/AuronCallNativeWrapper.java
b/auron-core/src/main/java/org/apache/auron/jni/AuronCallNativeWrapper.java
index 8060be19..661d8070 100644
--- a/auron-core/src/main/java/org/apache/auron/jni/AuronCallNativeWrapper.java
+++ b/auron-core/src/main/java/org/apache/auron/jni/AuronCallNativeWrapper.java
@@ -33,6 +33,11 @@ import org.apache.auron.protobuf.TaskDefinition;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * A wrapper class for calling native functions in the Auron project.
+ * It handles initialization, loading data batches, and error handling.
+ * Provides methods to interact with the native execution runtime and process
data batches.
+ */
public class AuronCallNativeWrapper {
private static final Logger LOG =
LoggerFactory.getLogger(AuronCallNativeWrapper.class);
diff --git
a/spark-extension/src/main/java/org/apache/spark/sql/auron/JniBridge.java
b/spark-extension/src/main/java/org/apache/spark/sql/auron/JniBridge.java
index 3078d487..3bea6e17 100644
--- a/spark-extension/src/main/java/org/apache/spark/sql/auron/JniBridge.java
+++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/JniBridge.java
@@ -36,38 +36,55 @@ import org.apache.spark.TaskContext$;
import org.apache.spark.sql.auron.memory.SparkOnHeapSpillManager$;
import org.apache.spark.sql.auron.util.TaskContextHelper$;
+/**
+ * This class has been deprecated and migrated to {@link
org.apache.auron.jni.JniBridge}.
+ * Will be removed in the future.
+ */
+@Deprecated
@SuppressWarnings("unused")
public class JniBridge {
+
+ @Deprecated
public static final ConcurrentHashMap<String, Object> resourcesMap = new
ConcurrentHashMap<>();
+ @Deprecated
public static native long callNative(long initNativeMemory, String
logLevel, AuronCallNativeWrapper wrapper);
+ @Deprecated
public static native boolean nextBatch(long ptr);
+ @Deprecated
public static native void finalizeNative(long ptr);
+ @Deprecated
public static native void onExit();
+ @Deprecated
public static ClassLoader getContextClassLoader() {
return Thread.currentThread().getContextClassLoader();
}
+ @Deprecated
public static void setContextClassLoader(ClassLoader cl) {
Thread.currentThread().setContextClassLoader(cl);
}
+ @Deprecated
public static Object getResource(String key) {
return resourcesMap.remove(key);
}
+ @Deprecated
public static TaskContext getTaskContext() {
return TaskContext$.MODULE$.get();
}
+ @Deprecated
public static OnHeapSpillManager getTaskOnHeapSpillManager() {
return SparkOnHeapSpillManager$.MODULE$.current();
}
+ @Deprecated
public static boolean isTaskRunning() {
TaskContext tc = getTaskContext();
if (tc == null) { // driver is always running
@@ -76,29 +93,35 @@ public class JniBridge {
return !tc.isCompleted() && !tc.isInterrupted();
}
+ @Deprecated
public static FSDataInputWrapper openFileAsDataInputWrapper(FileSystem fs,
String path) throws Exception {
// the path is a URI string, so we need to convert it to a URI object,
ref:
// org.apache.spark.paths.SparkPath.toPath
return FSDataInputWrapper$.MODULE$.wrap(fs.open(new Path(new
URI(path))));
}
+ @Deprecated
public static FSDataOutputWrapper createFileAsDataOutputWrapper(FileSystem
fs, String path) throws Exception {
return FSDataOutputWrapper$.MODULE$.wrap(fs.create(new Path(new
URI(path))));
}
+ @Deprecated
private static final List<BufferPoolMXBean> directMXBeans =
ManagementFactory.getPlatformMXBeans(BufferPoolMXBean.class);
+ @Deprecated
public static long getTotalMemoryLimited() {
return NativeHelper$.MODULE$.totalMemory();
}
+ @Deprecated
public static long getDirectMemoryUsed() {
return directMXBeans.stream()
.mapToLong(BufferPoolMXBean::getTotalCapacity)
.sum();
}
+ @Deprecated
public static String getDirectWriteSpillToDiskFile() {
return SparkEnv.get()
.blockManager()
@@ -108,6 +131,7 @@ public class JniBridge {
.getPath();
}
+ @Deprecated
public static void initNativeThread(ClassLoader cl, TaskContext tc) {
setContextClassLoader(cl);
TaskContext$.MODULE$.setTaskContext(tc);
@@ -115,6 +139,7 @@ public class JniBridge {
TaskContextHelper$.MODULE$.setHDFSCallerContext();
}
+ @Deprecated
public static AuronUDFWrapperContext getAuronUDFWrapperContext(ByteBuffer
udfSerialized) {
throw new UnsupportedOperationException("This API is designed to
support next-generation multi-engine.");
}
diff --git
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
index 5cf2131b..bf0918c4 100644
---
a/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
+++
b/spark-extension/src/main/scala/org/apache/spark/sql/auron/AuronCallNativeWrapper.scala
@@ -49,6 +49,11 @@ import org.apache.auron.protobuf.PartitionId
import org.apache.auron.protobuf.PhysicalPlanNode
import org.apache.auron.protobuf.TaskDefinition
+/**
+ * This class has been deprecated and migrated to {@link
+ * org.apache.auron.jni.AuronCallNativeWrapper}. Will be removed in the future.
+ */
+@Deprecated
case class AuronCallNativeWrapper(
nativePlan: PhysicalPlanNode,
partition: Partition,
@@ -94,6 +99,7 @@ case class AuronCallNativeWrapper(
false
}
+ @Deprecated
override def next(): InternalRow = {
val batchRow = batchRows(batchCurRowIdx)
batchCurRowIdx += 1
@@ -104,13 +110,16 @@ case class AuronCallNativeWrapper(
context.foreach(_.addTaskCompletionListener[Unit]((_: TaskContext) =>
close()))
context.foreach(_.addTaskFailureListener((_, _) => close()))
+ @Deprecated
def getRowIterator: Iterator[InternalRow] = {
CompletionIterator[InternalRow, Iterator[InternalRow]](rowIterator,
close())
}
+ @Deprecated
protected def getMetrics: MetricNode =
metrics
+ @Deprecated
protected def importSchema(ffiSchemaPtr: Long): Unit = {
Using.resource(ArrowSchema.wrap(ffiSchemaPtr)) { ffiSchema =>
arrowSchema = Data.importSchema(ROOT_ALLOCATOR, ffiSchema,
dictionaryProvider)
@@ -119,6 +128,7 @@ case class AuronCallNativeWrapper(
}
}
+ @Deprecated
protected def importBatch(ffiArrayPtr: Long): Unit = {
if (nativeRuntimePtr == 0) {
throw new RuntimeException("Native runtime is finalized")
@@ -137,10 +147,12 @@ case class AuronCallNativeWrapper(
}
}
+ @Deprecated
protected def setError(error: Throwable): Unit = {
this.error.set(error)
}
+ @Deprecated
protected def checkError(): Unit = {
val throwable = error.getAndSet(null)
if (throwable != null) {
@@ -149,6 +161,7 @@ case class AuronCallNativeWrapper(
}
}
+ @Deprecated
protected def getRawTaskDefinition: Array[Byte] = {
val partitionId: PartitionId = PartitionId
.newBuilder()
