This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/auron.git
The following commit(s) were added to refs/heads/master by this push:
new ee243adf [AURON #1409] Introduce Spark UI for auron (#1410)
ee243adf is described below
commit ee243adffca643608777c1e719fa089f702138ff
Author: guixiaowen <[email protected]>
AuthorDate: Mon Oct 13 10:04:33 2025 +0800
[AURON #1409] Introduce Spark UI for auron (#1410)
# Which issue does this PR close?
Closes #1409
# Rationale for this change
Display Auron's build information on the Spark UI.
# What changes are included in this PR?
1. Add a parameter to enable or disable this feature:
spark.auron.ui.enabled.
2. Record and parse Auron's build information.
3. Render the Spark UI page.
4. When initializing the Spark session, add the new page through the UI
settings.
# Are there any user-facing changes?
Add a new spark.auron.ui.enabled parameter.
# How was this patch tested?
**1. the Spark application is running**
When the Spark application is running, the Spark UI will display AURON SQL
/ DataFrame, showing Auron’s build information.
The information displayed in the Spark UI is as follows:
<img width="1485" height="665" alt="Image"
src="https://github.com/user-attachments/assets/1cf8dcab-b42b-45b6-b3a1-4c192ecacafe";
/>
**2. after the Spark application ends**
After the Spark application ends, the information can be viewed through the
Spark History Server.
You only need to package the auron-spark-ui project into a JAR and
integrate it with the Spark History Server, so that the information can be
viewed through the Spark History Server after the job finishes.
eg:
auron-spark-ui_2.12-7.0.0-SNAPSHOT.jar to shs jars
The information displayed in the Spark UI from Spark history server is as
follows:
<img width="1316" height="790" alt="Image"
src="https://github.com/user-attachments/assets/69ca3cd7-e0ad-4f3e-ac41-b995405b96d7";
/>
---------
Co-authored-by: guihuawen <[email protected]>
---
auron-build.sh | 25 +++++++++
auron-spark-ui/pom.xml | 56 +++++++++++++++++++
.../org.apache.spark.status.AppHistoryServerPlugin | 18 ++++++
.../org/apache/auron/spark/ui/AuronEvent.scala | 25 +++++++++
.../sql/execution/ui/AuronAllExecutionsPage.scala | 64 ++++++++++++++++++++++
.../spark/sql/execution/ui/AuronEventUtils.scala | 27 +++++++++
.../execution/ui/AuronSQLAppStatusListener.scala | 51 +++++++++++++++++
.../sql/execution/ui/AuronSQLAppStatusStore.scala | 34 ++++++++++++
.../execution/ui/AuronSQLHistoryServerPlugin.scala | 40 ++++++++++++++
.../spark/sql/execution/ui/AuronSQLTab.scala | 31 +++++++++++
.../org/apache/auron/common/AuronBuildInfo.scala | 64 ++++++++++++++++++++++
pom.xml | 1 +
spark-extension-shims-spark3/pom.xml | 5 ++
.../org/apache/spark/sql/auron/ShimsImpl.scala | 55 +++++++++++++++++--
.../spark/sql/auron/BuildInfoAuronSQLSuite.scala | 38 +++++++++++++
.../spark/sql/auron/BuildinfoInSparkUISuite.scala | 33 +++++++++++
spark-extension/pom.xml | 5 ++
.../java/org/apache/spark/sql/auron/AuronConf.java | 3 +
18 files changed, 570 insertions(+), 5 deletions(-)
diff --git a/auron-build.sh b/auron-build.sh
index 3e7b321b..230e6b40 100755
--- a/auron-build.sh
+++ b/auron-build.sh
@@ -275,6 +275,31 @@ fi
MVN_ARGS=("${CLEAN_ARGS[@]}" "${BUILD_ARGS[@]}")
+# -----------------------------------------------------------------------------
+# Write build information to auron-build-info.properties
+# -----------------------------------------------------------------------------
+BUILD_INFO_FILE="common/src/main/resources/auron-build-info.properties"
+mkdir -p "$(dirname "$BUILD_INFO_FILE")"
+
+JAVA_VERSION=$(java -version 2>&1 | head -n 1 | awk '{print $3}' | tr -d '"')
+PROJECT_VERSION=$(xmllint --xpath
"/*[local-name()='project']/*[local-name()='properties']/*[local-name()='project.version']/text()"
pom.xml)
+RUST_VERSION=$(rustc --version | awk '{print $2}')
+
+{
+ echo "spark.version=${SPARK_VER}"
+ echo "rust.version=${RUST_VERSION}"
+ echo "java.version=${JAVA_VERSION}"
+ echo "project.version=${PROJECT_VERSION}"
+ echo "scala.version=${SCALA_VER}"
+ echo "celeborn.version=${CELEBORN_VER}"
+ echo "uniffle.version=${UNIFFLE_VER}"
+ echo "paimon.version=${PAIMON_VER}"
+ echo "flink.version=${FLINK_VER}"
+ echo "build.timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+} > "$BUILD_INFO_FILE"
+
+echo "[INFO] Build info written to $BUILD_INFO_FILE"
+
# Execute Maven command
if [[ "$USE_DOCKER" == true ]]; then
# In Docker mode, use multi-threaded Maven build with -T8 for faster
compilation
diff --git a/auron-spark-ui/pom.xml b/auron-spark-ui/pom.xml
new file mode 100644
index 00000000..1a00142b
--- /dev/null
+++ b/auron-spark-ui/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.auron</groupId>
+ <artifactId>auron-parent_${scalaVersion}</artifactId>
+ <version>${project.version}</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>auron-spark-ui_${scalaVersion}</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scalaVersion}</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>prepare-test-jar</id>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>test-compile</phase>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
diff --git
a/auron-spark-ui/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
b/auron-spark-ui/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
new file mode 100644
index 00000000..4da3b151
--- /dev/null
+++
b/auron-spark-ui/src/main/resources/META-INF/services/org.apache.spark.status.AppHistoryServerPlugin
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.apache.spark.sql.execution.ui.AuronSQLHistoryServerPlugin
diff --git
a/auron-spark-ui/src/main/scala/org/apache/auron/spark/ui/AuronEvent.scala
b/auron-spark-ui/src/main/scala/org/apache/auron/spark/ui/AuronEvent.scala
new file mode 100644
index 00000000..b0dd2fdd
--- /dev/null
+++ b/auron-spark-ui/src/main/scala/org/apache/auron/spark/ui/AuronEvent.scala
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.spark.ui
+
+import scala.collection.mutable
+
+import org.apache.spark.scheduler.SparkListenerEvent
+
+sealed trait AuronEvent extends SparkListenerEvent {}
+
+case class AuronBuildInfoEvent(info: mutable.LinkedHashMap[String, String])
extends AuronEvent {}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala
new file mode 100644
index 00000000..c237557f
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronAllExecutionsPage.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import javax.servlet.http.HttpServletRequest
+
+import scala.xml.{Node, NodeSeq}
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.ui.{UIUtils, WebUIPage}
+
+private[ui] class AuronAllExecutionsPage(parent: AuronSQLTab) extends
WebUIPage("") with Logging {
+
+ private val sqlStore = parent.sqlStore
+
+ override def render(request: HttpServletRequest): Seq[Node] = {
+ val buildInfo = sqlStore.buildInfo()
+ val infos =
+ UIUtils.listingTable(propertyHeader, propertyRow, buildInfo.info,
fixedWidth = true)
+ val summary: NodeSeq =
+ <div>
+ <div>
+ <span class="collapse-sql-properties collapse-table"
+ onClick="collapseTable('collapse-sql-properties',
'sql-properties')">
+ <h4>
+ <span class="collapse-table-arrow arrow-open"></span>
+ <a>Auron Build Information</a>
+ </h4>
+ </span>
+ <div class="sql-properties collapsible-table">
+ {infos}
+ </div>
+ </div>
+ <br/>
+ </div>
+
+ UIUtils.headerSparkPage(request, "Auron", summary, parent)
+ }
+
+ private def propertyHeader = Seq("Name", "Value")
+
+ private def propertyRow(kv: (String, String)) = <tr>
+ <td>
+ {kv._1}
+ </td> <td>
+ {kv._2}
+ </td>
+ </tr>
+
+}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronEventUtils.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronEventUtils.scala
new file mode 100644
index 00000000..e4a359ce
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronEventUtils.scala
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.SparkContext
+
+import org.apache.auron.spark.ui.AuronEvent
+
+object AuronEventUtils {
+ def post(sc: SparkContext, event: AuronEvent): Unit = {
+ sc.listenerBus.post(event)
+ }
+}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
new file mode 100644
index 00000000..0da16d4f
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusListener.scala
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.{SparkConf, SparkContext}
+import org.apache.spark.internal.Logging
+import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent}
+import org.apache.spark.status.ElementTrackingStore
+
+import org.apache.auron.spark.ui.AuronBuildInfoEvent
+
+class AuronSQLAppStatusListener(conf: SparkConf, kvstore: ElementTrackingStore)
+ extends SparkListener
+ with Logging {
+
+ def getAuronBuildInfo(): Long = {
+ kvstore.count(classOf[AuronBuildInfoUIData])
+ }
+
+ private def onAuronBuildInfo(event: AuronBuildInfoEvent): Unit = {
+ val uiData = new AuronBuildInfoUIData(event.info.toSeq)
+ kvstore.write(uiData)
+ }
+
+ override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
+ case e: AuronBuildInfoEvent => onAuronBuildInfo(e)
+ case _ => // Ignore
+ }
+
+}
+object AuronSQLAppStatusListener {
+ def register(sc: SparkContext): Unit = {
+ val kvStore = sc.statusStore.store.asInstanceOf[ElementTrackingStore]
+ val listener = new AuronSQLAppStatusListener(sc.conf, kvStore)
+ sc.listenerBus.addToStatusQueue(listener)
+ }
+}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusStore.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusStore.scala
new file mode 100644
index 00000000..3fc4beb6
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLAppStatusStore.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import com.fasterxml.jackson.annotation.JsonIgnore
+import org.apache.spark.util.kvstore.{KVIndex, KVStore}
+
+class AuronSQLAppStatusStore(store: KVStore) {
+
+ def buildInfo(): AuronBuildInfoUIData = {
+ val kClass = classOf[AuronBuildInfoUIData]
+ store.read(kClass, kClass.getName)
+ }
+}
+
+class AuronBuildInfoUIData(val info: Seq[(String, String)]) {
+ @JsonIgnore
+ @KVIndex
+ def id: String = classOf[AuronBuildInfoUIData].getName()
+}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLHistoryServerPlugin.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLHistoryServerPlugin.scala
new file mode 100644
index 00000000..3ea7b5ad
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLHistoryServerPlugin.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.SparkConf
+import org.apache.spark.scheduler.SparkListener
+import org.apache.spark.status.{AppHistoryServerPlugin, ElementTrackingStore}
+import org.apache.spark.ui.SparkUI
+
+class AuronSQLHistoryServerPlugin extends AppHistoryServerPlugin {
+
+ override def createListeners(
+ conf: SparkConf,
+ store: ElementTrackingStore): Seq[SparkListener] = {
+ Seq(new AuronSQLAppStatusListener(conf, store))
+ }
+
+ override def setupUI(ui: SparkUI): Unit = {
+ val sqlStatusStore = new AuronSQLAppStatusStore(ui.store.store)
+ if (sqlStatusStore.buildInfo() != null) {
+ new AuronSQLTab(sqlStatusStore, ui)
+ }
+ }
+
+ override def displayOrder: Int = 0
+}
diff --git
a/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLTab.scala
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLTab.scala
new file mode 100644
index 00000000..fc685f73
--- /dev/null
+++
b/auron-spark-ui/src/main/scala/org/apache/spark/sql/execution/ui/AuronSQLTab.scala
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.ui
+
+import org.apache.spark.internal.Logging
+import org.apache.spark.ui.{SparkUI, SparkUITab}
+
+class AuronSQLTab(val sqlStore: AuronSQLAppStatusStore, sparkUI: SparkUI)
+ extends SparkUITab(sparkUI, "auron")
+ with Logging {
+
+ override val name = "AURON"
+
+ val parent = sparkUI
+ attachPage(new AuronAllExecutionsPage(this))
+ parent.attachTab(this)
+}
diff --git a/common/src/main/scala/org/apache/auron/common/AuronBuildInfo.scala
b/common/src/main/scala/org/apache/auron/common/AuronBuildInfo.scala
new file mode 100644
index 00000000..66cffa19
--- /dev/null
+++ b/common/src/main/scala/org/apache/auron/common/AuronBuildInfo.scala
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.auron.common
+
+import java.util.Properties
+
+import scala.util.Try
+
+object AuronBuildInfo {
+
+ private val buildFile = "auron-build-info.properties"
+ private val buildFileStream =
+ Thread.currentThread().getContextClassLoader.getResourceAsStream(buildFile)
+
+ if (buildFileStream == null) {
+ throw new Exception(s"Can not load the core build file: $buildFile")
+ }
+
+ private val unknown = "NULL"
+
+ private val props = new Properties()
+
+ try {
+ props.load(buildFileStream)
+ } finally {
+ Try(buildFileStream.close())
+ }
+
+ val VERSION_STRING: String = "PROJECT VERSION"
+ val JAVA_COMPILE_VERSION_STRING: String = "JAVA VERSION"
+ val SCALA_COMPILE_VERSION_STRING: String = "SCALA VERSION"
+ val SPARK_COMPILE_VERSION_STRING: String = "SPARK VERSION"
+ val RUST_COMPILE_VERSION_STRING: String = "RUST VERSION"
+ val CELEBORN_VERSION_STRING: String = "CELEBRON VERSION"
+ val UNIFFLE_VERSION_STRING: String = "UNIFFLE VERSION"
+ val PAIMON_VERSION_STRING: String = "PAIMON VERSION"
+ val FLINK_VERSION_STRING: String = "FLINK VERSION"
+ val BUILD_DATE_STRING: String = "BUILD TIMESTAMP"
+
+ val VERSION: String = props.getProperty("project.version", unknown)
+ val JAVA_COMPILE_VERSION: String = props.getProperty("java.version", unknown)
+ val SCALA_COMPILE_VERSION: String = props.getProperty("scala.version",
unknown)
+ val SPARK_COMPILE_VERSION: String = props.getProperty("spark.version",
unknown)
+ val RUST_COMPILE_VERSION: String = props.getProperty("rust.version", unknown)
+ val CELEBORN_VERSION: String = props.getProperty("celeborn.version", unknown)
+ val UNIFFLE_VERSION: String = props.getProperty("uniffle.version", unknown)
+ val PAIMON_VERSION: String = props.getProperty("paimon.version", unknown)
+ val FLINK_VERSION: String = props.getProperty("flink.version", unknown)
+ val BUILD_DATE: String = props.getProperty("build.timestamp", unknown)
+}
diff --git a/pom.xml b/pom.xml
index e63a5447..a717d710 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,6 +42,7 @@
<module>hadoop-shim</module>
<module>dev/mvn-build-helper/assembly</module>
<module>dev/mvn-build-helper/proto</module>
+ <module>auron-spark-ui</module>
</modules>
<properties>
diff --git a/spark-extension-shims-spark3/pom.xml
b/spark-extension-shims-spark3/pom.xml
index f230eb8b..267b6616 100644
--- a/spark-extension-shims-spark3/pom.xml
+++ b/spark-extension-shims-spark3/pom.xml
@@ -31,6 +31,11 @@
<description>Apache Auron Spark Extension Shims Project</description>
<dependencies>
+ <dependency>
+ <groupId>org.apache.auron</groupId>
+ <artifactId>auron-spark-ui_${scalaVersion}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.auron</groupId>
<artifactId>auron-common_${scalaVersion}</artifactId>
diff --git
a/spark-extension-shims-spark3/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
b/spark-extension-shims-spark3/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
index b28ff631..0ba63c8e 100644
---
a/spark-extension-shims-spark3/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
+++
b/spark-extension-shims-spark3/src/main/scala/org/apache/spark/sql/auron/ShimsImpl.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.auron
import java.io.File
import java.util.UUID
+import scala.collection.mutable
+
import org.apache.commons.lang3.reflect.FieldUtils
-import org.apache.spark.OneToOneDependency
-import org.apache.spark.ShuffleDependency
-import org.apache.spark.SparkEnv
-import org.apache.spark.SparkException
-import org.apache.spark.TaskContext
+import org.apache.spark.{OneToOneDependency, ShuffleDependency, SparkContext,
SparkEnv, SparkException, TaskContext}
import org.apache.spark.internal.Logging
import org.apache.spark.rdd.RDD
import org.apache.spark.scheduler.MapStatus
@@ -102,14 +100,18 @@ import
org.apache.spark.sql.execution.joins.auron.plan.NativeBroadcastJoinExec
import
org.apache.spark.sql.execution.joins.auron.plan.NativeShuffledHashJoinExecProvider
import
org.apache.spark.sql.execution.joins.auron.plan.NativeSortMergeJoinExecProvider
import org.apache.spark.sql.execution.metric.{SQLMetric,
SQLShuffleReadMetricsReporter}
+import org.apache.spark.sql.execution.ui.{AuronEventUtils,
AuronSQLAppStatusListener, AuronSQLAppStatusStore, AuronSQLTab}
import org.apache.spark.sql.hive.execution.InsertIntoHiveTable
import org.apache.spark.sql.types.DataType
import org.apache.spark.sql.types.IntegerType
import org.apache.spark.sql.types.StringType
+import org.apache.spark.status.ElementTrackingStore
import org.apache.spark.storage.BlockManagerId
import org.apache.spark.storage.FileSegment
import org.apache.auron.{protobuf => pb, sparkver}
+import org.apache.auron.common.AuronBuildInfo
+import org.apache.auron.spark.ui.AuronBuildInfoEvent
class ShimsImpl extends Shims with Logging {
@@ -146,6 +148,49 @@ class ShimsImpl extends Shims with Logging {
if (AuronConf.FORCE_SHUFFLED_HASH_JOIN.booleanConf()) {
logWarning(s"${AuronConf.FORCE_SHUFFLED_HASH_JOIN.key} is not supported
in $shimVersion")
}
+
+ }
+
+ // set Auron spark ui if spark.auron.ui.enabled is true
+ override def onApplyingExtension(): Unit = {
+ logInfo(
+ " onApplyingExtension get ui_enabled : " + SparkEnv.get.conf
+ .get(AuronConf.UI_ENABLED.key, "true"))
+
+ if (SparkEnv.get.conf.get(AuronConf.UI_ENABLED.key,
"true").equals("true")) {
+ val sparkContext = SparkContext.getOrCreate()
+ val kvStore =
sparkContext.statusStore.store.asInstanceOf[ElementTrackingStore]
+ val statusStore = new AuronSQLAppStatusStore(kvStore)
+ sparkContext.ui.foreach(new AuronSQLTab(statusStore, _))
+ logInfo(" onApplyingExtension add register ")
+ AuronSQLAppStatusListener.register(sparkContext)
+ postBuildInfoEvent(sparkContext)
+ }
+
+ }
+
+ private def postBuildInfoEvent(sparkContext: SparkContext): Unit = {
+ val auronBuildInfo = new mutable.LinkedHashMap[String, String]()
+ auronBuildInfo.put(AuronBuildInfo.VERSION_STRING, AuronBuildInfo.VERSION)
+ auronBuildInfo.put(
+ AuronBuildInfo.JAVA_COMPILE_VERSION_STRING,
+ AuronBuildInfo.JAVA_COMPILE_VERSION)
+ auronBuildInfo.put(
+ AuronBuildInfo.SCALA_COMPILE_VERSION_STRING,
+ AuronBuildInfo.SCALA_COMPILE_VERSION)
+ auronBuildInfo.put(
+ AuronBuildInfo.SPARK_COMPILE_VERSION_STRING,
+ AuronBuildInfo.SPARK_COMPILE_VERSION)
+ auronBuildInfo.put(
+ AuronBuildInfo.RUST_COMPILE_VERSION_STRING,
+ AuronBuildInfo.RUST_COMPILE_VERSION)
+ auronBuildInfo.put(AuronBuildInfo.CELEBORN_VERSION_STRING,
AuronBuildInfo.CELEBORN_VERSION)
+ auronBuildInfo.put(AuronBuildInfo.UNIFFLE_VERSION_STRING,
AuronBuildInfo.UNIFFLE_VERSION)
+ auronBuildInfo.put(AuronBuildInfo.PAIMON_VERSION_STRING,
AuronBuildInfo.PAIMON_VERSION)
+ auronBuildInfo.put(AuronBuildInfo.FLINK_VERSION_STRING,
AuronBuildInfo.FLINK_VERSION)
+ auronBuildInfo.put(AuronBuildInfo.BUILD_DATE_STRING,
AuronBuildInfo.BUILD_DATE)
+ val event = AuronBuildInfoEvent(auronBuildInfo)
+ AuronEventUtils.post(sparkContext, event)
}
override def createConvertToNativeExec(child: SparkPlan):
ConvertToNativeBase =
diff --git
a/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildInfoAuronSQLSuite.scala
b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildInfoAuronSQLSuite.scala
new file mode 100644
index 00000000..5ef9055c
--- /dev/null
+++
b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildInfoAuronSQLSuite.scala
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.auron
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.test.SharedSparkSession
+
+trait BuildInfoAuronSQLSuite extends SharedSparkSession {
+
+ override protected def sparkConf: SparkConf = {
+ super.sparkConf
+ .set("spark.sql.extensions",
"org.apache.spark.sql.auron.AuronSparkSessionExtension")
+ .set(
+ "spark.shuffle.manager",
+ "org.apache.spark.sql.execution.auron.shuffle.AuronShuffleManager")
+ .set("spark.memory.offHeap.enabled", "false")
+ .set("spark.eventLog.enabled", "true")
+ .set("spark.ui.enabled", "true")
+ .set("spark.auron.ui.enabled", "true")
+ .set("spark.ui.port", "4040")
+ .set("spark.auron.enable", "true")
+ }
+
+}
diff --git
a/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildinfoInSparkUISuite.scala
b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildinfoInSparkUISuite.scala
new file mode 100644
index 00000000..833ea719
--- /dev/null
+++
b/spark-extension-shims-spark3/src/test/scala/org/apache/spark/sql/auron/BuildinfoInSparkUISuite.scala
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.auron
+
+import org.apache.spark.sql.execution.ui.AuronSQLAppStatusListener
+
+class BuildinfoInSparkUISuite
+ extends org.apache.spark.sql.QueryTest
+ with BuildInfoAuronSQLSuite
+ with AuronSQLTestHelper {
+
+ test("test build info in spark UI ") {
+ val listeners =
spark.sparkContext.listenerBus.findListenersByClass[AuronSQLAppStatusListener]
+ assert(listeners.size === 1)
+ val listener = listeners(0)
+ assert(listener.getAuronBuildInfo() == 1)
+ }
+
+}
diff --git a/spark-extension/pom.xml b/spark-extension/pom.xml
index 55311428..59b8fc34 100644
--- a/spark-extension/pom.xml
+++ b/spark-extension/pom.xml
@@ -31,6 +31,11 @@
<description>Apache Auron Spark Extension Project</description>
<dependencies>
+ <dependency>
+ <groupId>org.apache.auron</groupId>
+ <artifactId>auron-spark-ui_${scalaVersion}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.auron</groupId>
<artifactId>auron-core</artifactId>
diff --git
a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java
b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java
index 1dd4d3f6..c16fce31 100644
--- a/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java
+++ b/spark-extension/src/main/java/org/apache/spark/sql/auron/AuronConf.java
@@ -21,6 +21,9 @@ import org.apache.spark.SparkEnv$;
@SuppressWarnings("unused")
public enum AuronConf {
+ // support spark.auron.ui.enabled
+ UI_ENABLED("spark.auron.ui.enabled", true),
+
/// suggested batch size for arrow batches.
BATCH_SIZE("spark.auron.batchSize", 10000),
