This is an automated email from the ASF dual-hosted git repository.
weitingchen pushed a commit to branch branch-1.2
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/branch-1.2 by this push:
new ae56ca51c5 [VL][1.2] Port #7121 #7448 (#7988)
ae56ca51c5 is described below
commit ae56ca51c5a08d8ebcf1708f91c9d4f1058d6640
Author: Wei-Ting Chen <[email protected]>
AuthorDate: Wed Nov 20 15:46:06 2024 +0800
[VL][1.2] Port #7121 #7448 (#7988)
* [GLUTEN-6748][CORE] Search stack trace to infer adaptive execution
context (#7121)
Closes #6748
* [Core] fix GH security issues (#7448)
Uncontrolled Resource Consumption vulnerability in Apache Commons IO.
The org.apache.commons.io.input.XmlStreamReader class may excessively
consume CPU resources when processing maliciously crafted input.
This issue affects Apache Commons IO: from 2.0 before 2.14.0.
Users are recommended to upgrade to version 2.14.0 or later, which fixes
the issue.
Signed-off-by: Yuan Zhou <[email protected]>
---------
Signed-off-by: Yuan Zhou <[email protected]>
Co-authored-by: PHILO-HE <[email protected]>
Co-authored-by: Yuan <[email protected]>
---
backends-clickhouse/pom.xml | 2 +-
backends-velox/pom.xml | 2 +-
gluten-core/pom.xml | 2 +-
.../columnar/enumerated/EnumeratedApplier.scala | 9 +--------
.../extension/columnar/heuristic/HeuristicApplier.scala | 9 +--------
.../extension/columnar/util/AdaptiveContext.scala | 17 +++++------------
gluten-ut/pom.xml | 2 +-
7 files changed, 11 insertions(+), 32 deletions(-)
diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml
index 8671efb985..0dcd20c929 100644
--- a/backends-clickhouse/pom.xml
+++ b/backends-clickhouse/pom.xml
@@ -177,7 +177,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.11.0</version>
+ <version>2.14.0</version>
<scope>provided</scope>
</dependency>
<!-- Fasterxml -->
diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index a4e0c07bd0..7d08a50c41 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -132,7 +132,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.11.0</version>
+ <version>2.14.0</version>
<scope>provided</scope>
</dependency>
<dependency>
diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml
index 9109c54c3a..5b59634967 100644
--- a/gluten-core/pom.xml
+++ b/gluten-core/pom.xml
@@ -195,7 +195,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.11.0</version>
+ <version>2.14.0</version>
<scope>provided</scope>
</dependency>
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala
index 519db966c2..b5460b995e 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala
@@ -44,14 +44,7 @@ class EnumeratedApplier(session: SparkSession)
extends ColumnarRuleApplier
with Logging
with LogLevelUtil {
- // An empirical value.
- private val aqeStackTraceIndex =
- if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) {
- 16
- } else {
- 14
- }
- private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex)
+ private val adaptiveContext = AdaptiveContext(session)
override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan =
PhysicalPlanSelector.maybe(session, plan) {
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala
index 03b2b66b09..a5980e456c 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/heuristic/HeuristicApplier.scala
@@ -39,14 +39,7 @@ class HeuristicApplier(session: SparkSession)
extends ColumnarRuleApplier
with Logging
with LogLevelUtil {
- // This is an empirical value, may need to be changed for supporting other
versions of spark.
- private val aqeStackTraceIndex =
- if (scala.util.Properties.releaseVersion.exists(_.startsWith("2.12"))) {
- 19
- } else {
- 17
- }
- private val adaptiveContext = AdaptiveContext(session, aqeStackTraceIndex)
+ private val adaptiveContext = AdaptiveContext(session)
override def apply(plan: SparkPlan, outputsColumnar: Boolean): SparkPlan = {
withTransformRules(transformRules(outputsColumnar)).apply(plan)
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala
index 4a9d69f8f0..41b88ffcb2 100644
---
a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala
+++
b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/util/AdaptiveContext.scala
@@ -33,8 +33,8 @@ sealed trait AdaptiveContext {
}
object AdaptiveContext {
- def apply(session: SparkSession, aqeStackTraceIndex: Int): AdaptiveContext =
- new AdaptiveContextImpl(session, aqeStackTraceIndex)
+ def apply(session: SparkSession): AdaptiveContext =
+ new AdaptiveContextImpl(session)
private val GLUTEN_IS_ADAPTIVE_CONTEXT = "gluten.isAdaptiveContext"
@@ -44,8 +44,7 @@ object AdaptiveContext {
private val localIsAdaptiveContextFlags: ThreadLocal[ListBuffer[Boolean]] =
ThreadLocal.withInitial(() => ListBuffer.empty[Boolean])
- private class AdaptiveContextImpl(session: SparkSession, aqeStackTraceIndex:
Int)
- extends AdaptiveContext {
+ private class AdaptiveContextImpl(session: SparkSession) extends
AdaptiveContext {
// Just for test use.
override def enableAdaptiveContext(): Unit = {
session.sparkContext.setLocalProperty(GLUTEN_IS_ADAPTIVE_CONTEXT, "true")
@@ -59,19 +58,13 @@ object AdaptiveContext {
override def setAdaptiveContext(): Unit = {
val traceElements = Thread.currentThread.getStackTrace
- assert(
- traceElements.length > aqeStackTraceIndex,
- s"The number of stack trace elements is expected to be more than
$aqeStackTraceIndex")
// ApplyColumnarRulesAndInsertTransitions is called by either
QueryExecution or
// AdaptiveSparkPlanExec. So by checking the stack trace, we can know
whether
- // columnar rule will be applied in adaptive execution context. This
part of code
- // needs to be carefully checked when supporting higher versions of
spark to make
- // sure the calling stack has not been changed.
+ // columnar rule will be applied in adaptive execution context.
localIsAdaptiveContextFlags
.get()
.prepend(
- traceElements(aqeStackTraceIndex).getClassName
- .equals(AdaptiveSparkPlanExec.getClass.getName))
+
traceElements.exists(_.getClassName.equals(AdaptiveSparkPlanExec.getClass.getName)))
}
override def resetAdaptiveContext(): Unit =
diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml
index e1fc405c64..9f4cca83b5 100644
--- a/gluten-ut/pom.xml
+++ b/gluten-ut/pom.xml
@@ -158,7 +158,7 @@
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
- <version>2.11.0</version>
+ <version>2.14.0</version>
<scope>provided</scope>
</dependency>
</dependencies>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
