This is an automated email from the ASF dual-hosted git repository.
jackylee pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4dcd2803fa [CORE] Post events until both spark ui and gluten ui are
enable (#9272)
4dcd2803fa is described below
commit 4dcd2803faa0599b50eac2ba3f999e652b6a0ab8
Author: Kaifei Yi <[email protected]>
AuthorDate: Mon Apr 14 12:04:40 2025 +0800
[CORE] Post events until both spark ui and gluten ui are enable (#9272)
---
.../main/scala/org/apache/gluten/GlutenPlugin.scala | 18 ++++++++----------
.../spark/sql/execution/GlutenFallbackReporter.scala | 6 +++---
gluten-ui/pom.xml | 6 ++++++
.../{GlutenEventUtils.scala => GlutenUIUtils.scala} | 20 ++++++++++++++++++--
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +++
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +++
.../spark/sql/gluten/GlutenFallbackSuite.scala | 3 +++
7 files changed, 44 insertions(+), 15 deletions(-)
diff --git a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
index f06330f925..d01a23712e 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/GlutenPlugin.scala
@@ -31,7 +31,7 @@ import org.apache.spark.api.plugin.{DriverPlugin,
ExecutorPlugin, PluginContext,
import org.apache.spark.internal.Logging
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.softaffinity.SoftAffinityListener
-import org.apache.spark.sql.execution.ui.{GlutenEventUtils,
GlutenSQLAppStatusListener}
+import org.apache.spark.sql.execution.ui.{GlutenSQLAppStatusListener,
GlutenUIUtils}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.StaticSQLConf.SPARK_SESSION_EXTENSIONS
import org.apache.spark.task.TaskResources
@@ -80,14 +80,12 @@ private[gluten] class GlutenDriverPlugin extends
DriverPlugin with Logging {
}
override def registerMetrics(appId: String, pluginContext: PluginContext):
Unit = {
- if (
- pluginContext.conf().getBoolean(GLUTEN_UI_ENABLED.key,
GLUTEN_UI_ENABLED.defaultValue.get)
- ) {
- _sc.foreach {
- sc =>
- GlutenEventUtils.attachUI(sc)
+ _sc.foreach {
+ sc =>
+ if (GlutenUIUtils.uiEnabled(sc)) {
+ GlutenUIUtils.attachUI(sc)
logInfo("Gluten SQL Tab has been attached.")
- }
+ }
}
}
@@ -131,9 +129,9 @@ private[gluten] class GlutenDriverPlugin extends
DriverPlugin with Logging {
"\n=============================================================="
)
logInfo(loggingInfo)
- if (sc.getConf.getBoolean(GLUTEN_UI_ENABLED.key,
GLUTEN_UI_ENABLED.defaultValue.get)) {
+ if (GlutenUIUtils.uiEnabled(sc)) {
val event = GlutenBuildInfoEvent(glutenBuildInfo.toMap)
- GlutenEventUtils.post(sc, event)
+ GlutenUIUtils.postEvent(sc, event)
}
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala
index 0c5f30a4b8..d7ffa42293 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/sql/execution/GlutenFallbackReporter.scala
@@ -25,7 +25,7 @@ import org.apache.gluten.logging.LogLevelUtil
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.catalyst.util.StringUtils.PlanStringConcat
-import org.apache.spark.sql.execution.ui.GlutenEventUtils
+import org.apache.spark.sql.execution.ui.GlutenUIUtils
/**
* This rule is used to collect all fallback reason.
@@ -40,7 +40,7 @@ case class GlutenFallbackReporter(glutenConf: GlutenConfig,
spark: SparkSession)
return plan
}
printFallbackReason(plan)
- if (glutenConf.glutenUiEnabled) {
+ if (GlutenUIUtils.uiEnabled(spark.sparkContext, Some(glutenConf))) {
postFallbackReason(plan)
}
plan
@@ -88,7 +88,7 @@ case class GlutenFallbackReporter(glutenConf: GlutenConfig,
spark: SparkSession)
concat.toString(),
fallbackNodeToReason
)
- GlutenEventUtils.post(sc, event)
+ GlutenUIUtils.postEvent(sc, event)
}
}
diff --git a/gluten-ui/pom.xml b/gluten-ui/pom.xml
index 94f8aee544..647a15e338 100644
--- a/gluten-ui/pom.xml
+++ b/gluten-ui/pom.xml
@@ -13,6 +13,12 @@
<name>Gluten UI</name>
<dependencies>
+ <dependency>
+ <groupId>org.apache.gluten</groupId>
+ <artifactId>spark-sql-columnar-shims-common</artifactId>
+ <version>${project.version}</version>
+ <scope>provided</scope>
+ </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
diff --git
a/gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenEventUtils.scala
b/gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenUIUtils.scala
similarity index 62%
rename from
gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenEventUtils.scala
rename to
gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenUIUtils.scala
index 245c678feb..eb9d56b33e 100644
---
a/gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenEventUtils.scala
+++
b/gluten-ui/src/main/scala/org/apache/spark/sql/execution/ui/GlutenUIUtils.scala
@@ -16,13 +16,29 @@
*/
package org.apache.spark.sql.execution.ui
+import org.apache.gluten.config.GlutenConfig
+import org.apache.gluten.config.GlutenConfig.GLUTEN_UI_ENABLED
import org.apache.gluten.events.GlutenEvent
import org.apache.spark.SparkContext
import org.apache.spark.status.ElementTrackingStore
-object GlutenEventUtils {
- def post(sc: SparkContext, event: GlutenEvent): Unit = {
+object GlutenUIUtils {
+
+ /**
+ * Check if enabled the gluten ui, Please note that, developer should pass
the gluten config if
+ * call from sql module to prevent misjudgment caused by users directly
setting sparkConf.
+ */
+ def uiEnabled(sc: SparkContext, glutenConfig: Option[GlutenConfig] = None):
Boolean = {
+ val glutenTabEnabled = if (glutenConfig.isDefined) {
+ glutenConfig.get.glutenUiEnabled
+ } else {
+ sc.getConf.getBoolean(GLUTEN_UI_ENABLED.key,
GLUTEN_UI_ENABLED.defaultValue.get)
+ }
+ sc.ui.isDefined && glutenTabEnabled
+ }
+
+ def postEvent(sc: SparkContext, event: GlutenEvent): Unit = {
sc.listenerBus.post(event)
}
diff --git
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
index 4fd7471648..e761f5a39f 100644
---
a/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
+++
b/gluten-ut/spark33/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
@@ -22,6 +22,7 @@ import org.apache.gluten.events.GlutenPlanFallbackEvent
import org.apache.gluten.execution.FileSourceScanExecTransformer
import org.apache.spark.SparkConf
+import org.apache.spark.internal.config.UI.UI_ENABLED
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
import org.apache.spark.sql.{GlutenSQLTestsTrait, Row}
import org.apache.spark.sql.execution.ProjectExec
@@ -37,6 +38,8 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with
AdaptiveSparkPlanHelp
super.sparkConf
.set(GlutenConfig.RAS_ENABLED.key, "false")
.set("spark.gluten.ui.enabled", "true")
+ // The gluten ui event test suite expects the spark ui to be enable
+ .set(UI_ENABLED, true)
}
testGluten("test fallback logging") {
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
index 2afb18120d..e53e28fa27 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
@@ -23,6 +23,7 @@ import
org.apache.gluten.execution.FileSourceScanExecTransformer
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
+import org.apache.spark.internal.config.UI.UI_ENABLED
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
import org.apache.spark.sql.{GlutenSQLTestsTrait, Row}
import org.apache.spark.sql.execution.ProjectExec
@@ -38,6 +39,8 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with
AdaptiveSparkPlanHelp
super.sparkConf
.set(GlutenConfig.RAS_ENABLED.key, "false")
.set("spark.gluten.ui.enabled", "true")
+ // The gluten ui event test suite expects the spark ui to be enable
+ .set(UI_ENABLED, true)
}
testGluten("test fallback logging") {
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
index 2afb18120d..e53e28fa27 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/gluten/GlutenFallbackSuite.scala
@@ -23,6 +23,7 @@ import
org.apache.gluten.execution.FileSourceScanExecTransformer
import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
+import org.apache.spark.internal.config.UI.UI_ENABLED
import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
import org.apache.spark.sql.{GlutenSQLTestsTrait, Row}
import org.apache.spark.sql.execution.ProjectExec
@@ -38,6 +39,8 @@ class GlutenFallbackSuite extends GlutenSQLTestsTrait with
AdaptiveSparkPlanHelp
super.sparkConf
.set(GlutenConfig.RAS_ENABLED.key, "false")
.set("spark.gluten.ui.enabled", "true")
+ // The gluten ui event test suite expects the spark ui to be enable
+ .set(UI_ENABLED, true)
}
testGluten("test fallback logging") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
