This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 61c0af52be [CORE] Avoid ClassNotFoundException when loading a shaded
protobuf class (#8996)
61c0af52be is described below
commit 61c0af52be89f8db4d3b0541db2c98dcf7e4141d
Author: PHILO-HE <[email protected]>
AuthorDate: Fri Mar 14 23:44:56 2025 +0800
[CORE] Avoid ClassNotFoundException when loading a shaded protobuf class
(#8996)
---
.../scala/org/apache/gluten/GlutenPlugin.scala | 2 +-
.../CodedInputStreamClassInitializer.scala | 26 +++++++++++++++++-----
2 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index c4f36b6728..41814492f3 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -270,7 +270,7 @@ private[gluten] class GlutenExecutorPlugin extends
ExecutorPlugin {
/** Initialize the executor plugin. */
override def init(ctx: PluginContext, extraConf: util.Map[String, String]):
Unit = {
- CodedInputStreamClassInitializer
+ CodedInputStreamClassInitializer.modifyDefaultRecursionLimitUnsafe
// Initialize Backend.
Component.sorted().foreach(_.onExecutorStart(ctx))
}
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/initializer/CodedInputStreamClassInitializer.scala
b/gluten-core/src/main/scala/org/apache/gluten/initializer/CodedInputStreamClassInitializer.scala
index 6dc3088bb3..dc09648a75 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/initializer/CodedInputStreamClassInitializer.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/initializer/CodedInputStreamClassInitializer.scala
@@ -16,6 +16,8 @@
*/
package org.apache.gluten.initializer
+import org.apache.spark.internal.Logging
+
import java.lang.reflect.Field
/**
@@ -23,20 +25,32 @@ import java.lang.reflect.Field
* the limit is hit for deeply nested plans. This is based on the fact that
the same class loader is
* used to load this class in the program, then this modification will really
take effect.
*/
-object CodedInputStreamClassInitializer {
- {
+object CodedInputStreamClassInitializer extends Logging {
+ private val newDefaultRecursionLimit = 100000
+
+ def modifyDefaultRecursionLimitUnsafe: Unit = {
try {
// scalastyle:off classforname
- // Use the shaded class name.
val clazz: Class[_] =
-
Class.forName("org.apache.gluten.shaded.com.google.protobuf.CodedInputStream")
+ try {
+ // Use the shaded class name.
+
Class.forName("org.apache.gluten.shaded.com.google.protobuf.CodedInputStream")
+ } catch {
+ // The above class is shaded in final package phase (see
package/pom.xml).
+ // If ClassNotFoundException happens, e.g., in mvn test, load the
original class instead.
+ case _: ClassNotFoundException =>
+ Class.forName("com.google.protobuf.CodedInputStream")
+ }
// scalastyle:on classforname
val field: Field = clazz.getDeclaredField("defaultRecursionLimit")
field.setAccessible(true)
// Enlarge defaultRecursionLimit whose original value is 100.
- field.setInt(null, 100000)
+ field.setInt(null, newDefaultRecursionLimit)
+ logInfo(
+ s"The defaultRecursionLimit in protobuf has been increased to
$newDefaultRecursionLimit")
} catch {
- case e: Exception => e.printStackTrace()
+ case e: Exception =>
+ log.error("Failed to modify the DefaultRecursionLimit in protobuf: " +
e.getMessage)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
