This is an automated email from the ASF dual-hosted git repository.
hongze pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 30f301adb2 [CORE] Remove library unloading API from JniLibLoader as
unused (#9277)
30f301adb2 is described below
commit 30f301adb2b2596cac0fbf530a9ebe9430126bb4
Author: Hongze Zhang <[email protected]>
AuthorDate: Thu Apr 10 11:16:03 2025 +0100
[CORE] Remove library unloading API from JniLibLoader as unused (#9277)
---
.../backendsapi/clickhouse/CHListenerApi.scala | 4 +-
.../execution/benchmarks/CHSqlBasedBenchmark.scala | 4 --
.../backendsapi/velox/VeloxListenerApi.scala | 16 ++---
.../gluten/utils/SharedLibraryLoaderCentos7.scala | 37 +++++-----
.../gluten/utils/SharedLibraryLoaderCentos8.scala | 45 ++++++------
.../gluten/utils/SharedLibraryLoaderCentos9.scala | 45 ++++++------
.../gluten/utils/SharedLibraryLoaderDebian11.scala | 48 ++++++-------
.../gluten/utils/SharedLibraryLoaderDebian12.scala | 60 ++++++++--------
.../utils/SharedLibraryLoaderUbuntu2004.scala | 80 +++++++++++-----------
.../utils/SharedLibraryLoaderUbuntu2204.scala | 50 +++++++-------
.../java/org/apache/gluten/jni/JniLibLoader.java | 78 +++------------------
.../benchmarks/ParquetReadBenchmark.scala | 4 --
.../benchmarks/ParquetReadBenchmark.scala | 4 --
.../benchmarks/ParquetReadBenchmark.scala | 4 --
.../benchmarks/ParquetReadBenchmark.scala | 4 --
15 files changed, 193 insertions(+), 290 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
index 52e0faed32..5e62126ff8 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHListenerApi.scala
@@ -82,10 +82,10 @@ class CHListenerApi extends ListenerApi with Logging {
"Please set spark.gluten.sql.columnar.libpath to enable clickhouse
backend")
}
if (isDriver) {
- JniLibLoader.loadFromPath(libPath, true)
+ JniLibLoader.loadFromPath(libPath)
} else {
val executorLibPath =
conf.get(GlutenConfig.GLUTEN_EXECUTOR_LIB_PATH.key, libPath)
- JniLibLoader.loadFromPath(executorLibPath, true)
+ JniLibLoader.loadFromPath(executorLibPath)
}
CHListenerApi.addShutdownHook
// Add configs
diff --git
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
index 2a9a2b233a..76bcb552fc 100644
---
a/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/spark/sql/execution/benchmarks/CHSqlBasedBenchmark.scala
@@ -16,9 +16,6 @@
*/
package org.apache.spark.sql.execution.benchmarks
-import org.apache.gluten.config.GlutenConfig
-import org.apache.gluten.jni.JniLibLoader
-
import org.apache.spark.SparkConf
import org.apache.spark.sql.delta.DeltaLog
import org.apache.spark.sql.execution.benchmark.SqlBasedBenchmark
@@ -55,7 +52,6 @@ trait CHSqlBasedBenchmark extends SqlBasedBenchmark {
override def afterAll(): Unit = {
DeltaLog.clearCache()
-
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
// Wait for Ctrl+C, convenient for seeing Spark UI
// Thread.sleep(600000)
super.afterAll()
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 109f3015a6..6f981613c5 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -41,7 +41,7 @@ import
org.apache.spark.sql.execution.datasources.GlutenWriterColumnarRules
import
org.apache.spark.sql.execution.datasources.velox.{VeloxParquetWriterInjects,
VeloxRowSplitter}
import org.apache.spark.sql.expression.UDFResolver
import org.apache.spark.sql.internal.{GlutenConfigUtil, StaticSQLConf}
-import org.apache.spark.util.{SparkDirectoryUtil, SparkResourceUtil,
SparkShutdownManagerUtil}
+import org.apache.spark.util.{SparkDirectoryUtil, SparkResourceUtil}
import org.apache.commons.lang3.StringUtils
@@ -149,7 +149,6 @@ class VeloxListenerApi extends ListenerApi with Logging {
override def onExecutorShutdown(): Unit = shutdown()
private def initialize(conf: SparkConf, isDriver: Boolean): Unit = {
- addShutdownHook
// Sets this configuration only once, since not undoable.
// DebugInstance should be created first.
if (conf.getBoolean(GlutenConfig.DEBUG_KEEP_JNI_WORKSPACE.key,
defaultValue = false)) {
@@ -201,11 +200,11 @@ class VeloxListenerApi extends ListenerApi with Logging {
// Load backend libraries.
val libPath = conf.get(GlutenConfig.GLUTEN_LIB_PATH.key, StringUtils.EMPTY)
if (StringUtils.isNotBlank(libPath)) { // Path based load. Ignore all
other loadees.
- JniLibLoader.loadFromPath(libPath, false)
+ JniLibLoader.loadFromPath(libPath)
} else {
val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME.key, "gluten")
- loader.load(s"$platformLibDir/${System.mapLibraryName(baseLibName)}",
false)
-
loader.load(s"$platformLibDir/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}",
false)
+ loader.load(s"$platformLibDir/${System.mapLibraryName(baseLibName)}")
+
loader.load(s"$platformLibDir/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}")
}
// Initial native backend with configurations.
@@ -251,11 +250,4 @@ object VeloxListenerApi {
private def inLocalMode(conf: SparkConf): Boolean = {
SparkResourceUtil.isLocalMaster(conf)
}
-
- private def addShutdownHook: Unit = {
- SparkShutdownManagerUtil.addHookForLibUnloading(
- () => {
- JniLibLoader.forceUnloadAll
- })
- }
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
index b9d90d589c..56badac907 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala
@@ -20,25 +20,22 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
- loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
- loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink(
- "libboost_program_options.so.1.84.0",
- "libboost_program_options.so",
- false)
- loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libdouble-conversion.so.1",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
- loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- loader.loadAndCreateLink("libre2.so.10", "libre2.so", false)
- loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false)
- loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false)
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
+ loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so")
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.1",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so")
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so")
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
+ loader.loadAndCreateLink("libre2.so.10", "libre2.so")
+ loader.loadAndCreateLink("libzstd.so.1", "libzstd.so")
+ loader.loadAndCreateLink("liblz4.so.1", "liblz4.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
index dbb7d59f88..d049b73cc3 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala
@@ -20,29 +20,26 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
- loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
- loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
- loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false)
- loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false)
- loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink(
- "libboost_program_options.so.1.84.0",
- "libboost_program_options.so",
- false)
- loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
- loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false)
- loader.loadAndCreateLink("libidn.so.11", "libidn.so", false)
- loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- loader.loadAndCreateLink("libre2.so.0", "libre2.so", false)
- loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
+ loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
+ loader.loadAndCreateLink("libicudata.so.60", "libicudata.so")
+ loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so")
+ loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so")
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so")
+ loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so")
+ loader.loadAndCreateLink("libidn.so.11", "libidn.so")
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so")
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
+ loader.loadAndCreateLink("libre2.so.0", "libre2.so")
+ loader.loadAndCreateLink("libsodium.so.23", "libsodium.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
index f633a79f8f..fa50c2fcc6 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala
@@ -20,29 +20,26 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libboost_atomic.so.1.84.0",
"libboost_atomic.so", false)
- loader.loadAndCreateLink("libboost_thread.so.1.84.0",
"libboost_thread.so", false)
- loader.loadAndCreateLink("libboost_system.so.1.84.0",
"libboost_system.so", false)
- loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
- loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
- loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink(
- "libboost_program_options.so.1.84.0",
- "libboost_program_options.so",
- false)
- loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
- loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false)
- loader.loadAndCreateLink("libidn.so.12", "libidn.so", false)
- loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
- loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false)
+ loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so")
+ loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so")
+ loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so")
+ loader.loadAndCreateLink("libicudata.so.67", "libicudata.so")
+ loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so")
+ loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libboost_program_options.so.1.84.0",
"libboost_program_options.so")
+ loader.loadAndCreateLink("libboost_filesystem.so.1.84.0",
"libboost_filesystem.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so")
+ loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
+ loader.loadAndCreateLink("libidn.so.12", "libidn.so")
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so")
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so")
+ loader.loadAndCreateLink("libsodium.so.23", "libsodium.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
index 4f9b6fdd72..611ca9091f 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala
@@ -20,29 +20,29 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderDebian11 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false)
- loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
- loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false)
- loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
- loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
- loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false)
- loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
- loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
- loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libicudata.so.67", "libicudata.so")
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so")
+ loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so")
+ loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so")
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
+ loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
+ loader.loadAndCreateLink("libssh2.so.1", "libssh2.so")
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so")
+ loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so")
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so")
+ loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so")
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
index e3967eea32..9f7650ed1f 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala
@@ -20,35 +20,35 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderDebian12 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false)
- loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false)
- loader.loadAndCreateLink("libssl.so.3", "libssl.so", false)
- loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false)
- loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false)
- loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false)
- loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false)
- loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false)
- loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false)
- loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false)
- loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
- loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false)
- loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false)
- loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false)
- loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libglog.so.1", "libglog.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
+ loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so")
+ loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so")
+ loader.loadAndCreateLink("libssl.so.3", "libssl.so")
+ loader.loadAndCreateLink("libicudata.so.72", "libicudata.so")
+ loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so")
+ loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so")
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
+ loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so")
+ loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so")
+ loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so")
+ loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so")
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
+ loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
+ loader.loadAndCreateLink("libssh2.so.1", "libssh2.so")
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so")
+ loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so")
+ loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so")
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libglog.so.1", "libglog.so")
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
index 6d9271e9e3..6f463f3427 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala
@@ -20,45 +20,45 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libroken.so.18", "libroken.so", false)
- loader.loadAndCreateLink("libasn1.so.8", "libasn1.so", false)
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so",
false)
- loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false)
- loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false)
- loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false)
- loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- loader.loadAndCreateLink("libnettle.so.7", "libnettle.so", false)
- loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false)
- loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- loader.loadAndCreateLink("libssh.so.4", "libssh.so", false)
- loader.loadAndCreateLink("libssl.so.1.1", "libssl.so", false)
- loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false)
- loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- loader.loadAndCreateLink("libwind.so.0", "libwind.so", false)
- loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false)
- loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false)
- loader.loadAndCreateLink("libhx509.so.5", "libhx509.so", false)
- loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false)
- loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false)
- loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false)
- loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false)
- loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
- loader.loadAndCreateLink("libidn.so.11", "libidn.so", false)
- loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- loader.loadAndCreateLink("libicudata.so.66", "libicudata.so", false)
- loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false)
- loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
- loader.loadAndCreateLink("libre2.so.5", "libre2.so", false)
- loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false)
+ loader.loadAndCreateLink("libroken.so.18", "libroken.so")
+ loader.loadAndCreateLink("libasn1.so.8", "libasn1.so")
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
+ loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so")
+ loader.loadAndCreateLink("libpsl.so.5", "libpsl.so")
+ loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so")
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
+ loader.loadAndCreateLink("libnettle.so.7", "libnettle.so")
+ loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so")
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
+ loader.loadAndCreateLink("libssh.so.4", "libssh.so")
+ loader.loadAndCreateLink("libssl.so.1.1", "libssl.so")
+ loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so")
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
+ loader.loadAndCreateLink("libwind.so.0", "libwind.so")
+ loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so")
+ loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so")
+ loader.loadAndCreateLink("libhx509.so.5", "libhx509.so")
+ loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so")
+ loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so")
+ loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so")
+ loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so")
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so")
+ loader.loadAndCreateLink("libidn.so.11", "libidn.so")
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so")
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
+ loader.loadAndCreateLink("libicudata.so.66", "libicudata.so")
+ loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so")
+ loader.loadAndCreateLink("libxml2.so.2", "libxml2.so")
+ loader.loadAndCreateLink("libre2.so.5", "libre2.so")
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
+ loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so")
}
}
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
index 95f7db5655..029ddf45fa 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala
@@ -20,30 +20,30 @@ import org.apache.gluten.jni.JniLibLoader
class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader {
override def loadLib(loader: JniLibLoader): Unit = {
- loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so", false)
- loader.loadAndCreateLink("libicudata.so.70", "libicudata.so", false)
- loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false)
- loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false)
- loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so",
false)
- loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false)
- loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false)
- loader.loadAndCreateLink("libssh.so.4", "libssh.so", false)
- loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false)
- loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false)
- loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false)
- loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false)
- loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so", false)
- loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false)
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false)
- loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false)
- loader.loadAndCreateLink("libglog.so.0", "libglog.so", false)
- loader.loadAndCreateLink("libidn.so.12", "libidn.so", false)
- loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false)
- loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false)
- loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false)
- loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false)
- loader.loadAndCreateLink("libre2.so.9", "libre2.so", false)
- loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false)
- loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false)
+ loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
+ loader.loadAndCreateLink("libicudata.so.70", "libicudata.so")
+ loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so")
+ loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so")
+ loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so")
+ loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
+ loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
+ loader.loadAndCreateLink("libssh.so.4", "libssh.so")
+ loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
+ loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so")
+ loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so")
+ loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
+ loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
+ loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
+ loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
+ loader.loadAndCreateLink("libglog.so.0", "libglog.so")
+ loader.loadAndCreateLink("libidn.so.12", "libidn.so")
+ loader.loadAndCreateLink("libntlm.so.0", "libntlm.so")
+ loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so")
+ loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so")
+ loader.loadAndCreateLink("libxml2.so.2", "libxml2.so")
+ loader.loadAndCreateLink("libre2.so.9", "libre2.so")
+ loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so")
+ loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so")
}
}
diff --git a/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java
b/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java
index 72898823f5..00037cd228 100644
--- a/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java
+++ b/gluten-core/src/main/java/org/apache/gluten/jni/JniLibLoader.java
@@ -25,25 +25,16 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.Method;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashSet;
-import java.util.List;
import java.util.Set;
-import java.util.Vector;
public class JniLibLoader {
private static final Logger LOG =
LoggerFactory.getLogger(JniLibLoader.class);
private static final Set<String> LOADED_LIBRARY_PATHS = new HashSet<>();
- private static final Set<String> REQUIRE_UNLOAD_LIBRARY_PATHS = new
LinkedHashSet<>();
private final String workDir;
private final Set<String> loadedLibraries = new HashSet<>();
@@ -52,13 +43,6 @@ public class JniLibLoader {
this.workDir = workDir;
}
- public static synchronized void forceUnloadAll() {
- List<String> loaded;
- loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS);
- Collections.reverse(loaded); // use reversed order to unload
- loaded.forEach(JniLibLoader::unloadFromPath);
- }
-
private static String toRealPath(String libPath) {
String realPath = libPath;
try {
@@ -72,7 +56,7 @@ public class JniLibLoader {
}
}
- private static void loadFromPath0(String libPath, boolean requireUnload) {
+ private static void loadFromPath0(String libPath) {
libPath = toRealPath(libPath);
if (LOADED_LIBRARY_PATHS.contains(libPath)) {
LOG.debug("Library in path {} has already been loaded, skipping",
libPath);
@@ -81,66 +65,24 @@ public class JniLibLoader {
LOADED_LIBRARY_PATHS.add(libPath);
LOG.info("Library {} has been loaded using path-loading method",
libPath);
}
- if (requireUnload) {
- REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath);
- }
}
- public static synchronized void loadFromPath(String libPath, boolean
requireUnload) {
+ public static synchronized void loadFromPath(String libPath) {
final File file = new File(libPath);
if (!file.isFile() || !file.exists()) {
throw new GlutenException("library at path: " + libPath + " is not a
file or does not exist");
}
- loadFromPath0(file.getAbsolutePath(), requireUnload);
- }
-
- public static synchronized void unloadFromPath(String libPath) {
- if (!LOADED_LIBRARY_PATHS.remove(libPath)) {
- LOG.warn("Library {} was not loaded or already unloaded:", libPath);
- return;
- }
- LOG.info("Starting unload library path: {} ", libPath);
- REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath);
- try {
- ClassLoader classLoader = JniLibLoader.class.getClassLoader();
- Field field = ClassLoader.class.getDeclaredField("nativeLibraries");
- field.setAccessible(true);
- Vector<Object> libs = (Vector<Object>) field.get(classLoader);
- Iterator<Object> it = libs.iterator();
- while (it.hasNext()) {
- Object object = it.next();
- Field[] fs = object.getClass().getDeclaredFields();
- for (int k = 0; k < fs.length; k++) {
- if (fs[k].getName().equals("name")) {
- fs[k].setAccessible(true);
- String verbosePath = fs[k].get(object).toString();
- File verboseFile = new File(verbosePath);
- String verboseFileName = verboseFile.getName();
- File libFile = new File(libPath);
- String libFileName = libFile.getName();
-
- if (verboseFileName.equals(libFileName)) {
- LOG.info("Finalizing library file: {}", libFileName);
- Method finalize =
object.getClass().getDeclaredMethod("finalize");
- finalize.setAccessible(true);
- finalize.invoke(object);
- }
- }
- }
- }
- } catch (Throwable th) {
- LOG.error("Unload native library error: ", th);
- }
+ loadFromPath0(file.getAbsolutePath());
}
- public synchronized void load(String libPath, boolean requireUnload) {
+ public synchronized void load(String libPath) {
try {
if (loadedLibraries.contains(libPath)) {
LOG.debug("Library {} has already been loaded, skipping", libPath);
return;
}
File file = moveToWorkDir(workDir, libPath);
- loadWithLink(file.getAbsolutePath(), null, requireUnload);
+ loadWithLink(file.getAbsolutePath(), null);
loadedLibraries.add(libPath);
LOG.info("Successfully loaded library {}", libPath);
} catch (IOException e) {
@@ -148,14 +90,13 @@ public class JniLibLoader {
}
}
- public synchronized void loadAndCreateLink(
- String libPath, String linkName, boolean requireUnload) {
+ public synchronized void loadAndCreateLink(String libPath, String linkName) {
try {
if (loadedLibraries.contains(libPath)) {
LOG.debug("Library {} has already been loaded, skipping", libPath);
}
File file = moveToWorkDir(workDir, libPath);
- loadWithLink(file.getAbsolutePath(), linkName, requireUnload);
+ loadWithLink(file.getAbsolutePath(), linkName);
loadedLibraries.add(libPath);
LOG.info("Successfully loaded library {}", libPath);
} catch (IOException e) {
@@ -186,9 +127,8 @@ public class JniLibLoader {
return temp;
}
- private void loadWithLink(String libPath, String linkName, boolean
requireUnload)
- throws IOException {
- loadFromPath0(libPath, requireUnload);
+ private void loadWithLink(String libPath, String linkName) throws
IOException {
+ loadFromPath0(libPath);
LOG.info("Library {} has been loaded", libPath);
if (linkName != null) {
Path target = Paths.get(libPath);
diff --git
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index aa561f17f0..93aa0f2e7d 100644
---
a/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark32/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.benchmarks
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
-import org.apache.gluten.jni.JniLibLoader
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
@@ -223,9 +222,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
override def afterAll(): Unit = {
- if (BackendTestUtils.isCHBackendLoaded()) {
-
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
- }
super.afterAll()
}
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index 517f17bba9..8fc9ba8c9c 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.benchmarks
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
-import org.apache.gluten.jni.JniLibLoader
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
@@ -223,9 +222,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
override def afterAll(): Unit = {
- if (BackendTestUtils.isCHBackendLoaded()) {
-
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
- }
super.afterAll()
}
}
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index 62eb3ec8a3..e6e8124e02 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -20,7 +20,6 @@ import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
-import org.apache.gluten.jni.JniLibLoader
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
@@ -225,9 +224,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
override def afterAll(): Unit = {
- if (BackendTestUtils.isCHBackendLoaded()) {
-
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
- }
super.afterAll()
}
}
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
index 517f17bba9..8fc9ba8c9c 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/execution/benchmarks/ParquetReadBenchmark.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.execution.benchmarks
import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{FileSourceScanExecTransformer,
WholeStageTransformer}
import org.apache.gluten.extension.columnar.transition.Transitions
-import org.apache.gluten.jni.JniLibLoader
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
@@ -223,9 +222,6 @@ object ParquetReadBenchmark extends SqlBasedBenchmark {
}
override def afterAll(): Unit = {
- if (BackendTestUtils.isCHBackendLoaded()) {
-
JniLibLoader.unloadFromPath(spark.conf.get(GlutenConfig.GLUTEN_LIB_PATH.key))
- }
super.afterAll()
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
