This is an automated email from the ASF dual-hosted git repository. taiyang-li pushed a commit to branch fake_add_bolt_backend in repository https://gitbox.apache.org/repos/asf/gluten.git
commit bbaf1c218edbcb8e5165884f883606c0ba66d5e7 Author: liyang.127 <[email protected]> AuthorDate: Mon Jun 29 21:01:03 2026 +0800 [GLUTEN][CORE] Extract shared write-file metrics updater Introduce the backends-core module with a shared write-file metrics base so Velox and ClickHouse can reuse the same common SQLMetric updates while keeping backend-specific metrics in thin wrappers. --- backends-clickhouse/pom.xml | 6 ++ .../gluten/metrics/WriteFilesMetricsUpdater.scala | 8 +- backends-core/pom.xml | 95 ++++++++++++++++++++++ .../shared/SharedWriteFilesMetricsUpdater.scala | 18 ++-- backends-velox/pom.xml | 6 ++ .../gluten/metrics/WriteFilesMetricsUpdater.scala | 8 +- pom.xml | 1 + 7 files changed, 128 insertions(+), 14 deletions(-) diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index 816aeab6f8..1743036681 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -46,6 +46,12 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.gluten</groupId> + <artifactId>backends-core</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala index 5a04b40433..91cfbbccfd 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala @@ -16,15 +16,17 @@ */ package org.apache.gluten.metrics +import org.apache.gluten.metrics.shared.SharedWriteFilesMetricsUpdater + import org.apache.spark.sql.execution.metric.SQLMetric -class WriteFilesMetricsUpdater(val metrics: Map[String, SQLMetric]) extends MetricsUpdater { +class WriteFilesMetricsUpdater(override val metrics: Map[String, SQLMetric]) + extends SharedWriteFilesMetricsUpdater(metrics) { override def updateNativeMetrics(opMetrics: IOperatorMetrics): Unit = { if (opMetrics != null) { val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics] - metrics("physicalWrittenBytes") += operatorMetrics.physicalWrittenBytes - metrics("numWrittenFiles") += operatorMetrics.numWrittenFiles + updateSharedMetrics(operatorMetrics.physicalWrittenBytes, operatorMetrics.numWrittenFiles) } } } diff --git a/backends-core/pom.xml b/backends-core/pom.xml new file mode 100644 index 0000000000..6e3fa13b70 --- /dev/null +++ b/backends-core/pom.xml @@ -0,0 +1,95 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one or more + ~ contributor license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright ownership. + ~ The ASF licenses this file to You under the Apache License, Version 2.0 + ~ (the "License"); you may not use this file except in compliance with + ~ the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.gluten</groupId> + <artifactId>gluten-parent</artifactId> + <version>1.8.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <artifactId>backends-core</artifactId> + <packaging>jar</packaging> + <name>Gluten Backends Core</name> + + <dependencies> + <dependency> + <groupId>org.apache.gluten</groupId> + <artifactId>gluten-substrait</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-sql_${scala.binary.version}</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-core_${scala.binary.version}</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-catalyst_${scala.binary.version}</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.scala-lang</groupId> + <artifactId>scala-library</artifactId> + <version>${scala.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>net.alchim31.maven</groupId> + <artifactId>scala-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.scalastyle</groupId> + <artifactId>scalastyle-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + </plugin> + <plugin> + <groupId>com.diffplug.spotless</groupId> + <artifactId>spotless-maven-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>flatten-maven-plugin</artifactId> + </plugin> + </plugins> + <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> + <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> + </build> +</project> diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala b/backends-core/src/main/scala/org/apache/gluten/metrics/shared/SharedWriteFilesMetricsUpdater.scala similarity index 65% copy from backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala copy to backends-core/src/main/scala/org/apache/gluten/metrics/shared/SharedWriteFilesMetricsUpdater.scala index 5a04b40433..0576101c9e 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala +++ b/backends-core/src/main/scala/org/apache/gluten/metrics/shared/SharedWriteFilesMetricsUpdater.scala @@ -14,17 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.gluten.metrics +package org.apache.gluten.metrics.shared + +import org.apache.gluten.metrics.MetricsUpdater import org.apache.spark.sql.execution.metric.SQLMetric -class WriteFilesMetricsUpdater(val metrics: Map[String, SQLMetric]) extends MetricsUpdater { +abstract class SharedWriteFilesMetricsUpdater(val metrics: Map[String, SQLMetric]) + extends MetricsUpdater { - override def updateNativeMetrics(opMetrics: IOperatorMetrics): Unit = { - if (opMetrics != null) { - val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics] - metrics("physicalWrittenBytes") += operatorMetrics.physicalWrittenBytes - metrics("numWrittenFiles") += operatorMetrics.numWrittenFiles - } + final protected def updateSharedMetrics( + physicalWrittenBytes: Long, + numWrittenFiles: Long): Unit = { + metrics("physicalWrittenBytes") += physicalWrittenBytes + metrics("numWrittenFiles") += numWrittenFiles } } diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index a8d032a38b..3e81895ea7 100644 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -40,6 +40,12 @@ <version>${project.version}</version> <scope>compile</scope> </dependency> + <dependency> + <groupId>org.apache.gluten</groupId> + <artifactId>backends-core</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> diff --git a/backends-velox/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala b/backends-velox/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala index 7dc0ca880c..9df6a298d7 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/metrics/WriteFilesMetricsUpdater.scala @@ -16,17 +16,19 @@ */ package org.apache.gluten.metrics +import org.apache.gluten.metrics.shared.SharedWriteFilesMetricsUpdater + import org.apache.spark.sql.execution.metric.SQLMetric -class WriteFilesMetricsUpdater(val metrics: Map[String, SQLMetric]) extends MetricsUpdater { +class WriteFilesMetricsUpdater(override val metrics: Map[String, SQLMetric]) + extends SharedWriteFilesMetricsUpdater(metrics) { override def updateNativeMetrics(opMetrics: IOperatorMetrics): Unit = { if (opMetrics != null) { val operatorMetrics = opMetrics.asInstanceOf[OperatorMetrics] - metrics("physicalWrittenBytes") += operatorMetrics.physicalWrittenBytes + updateSharedMetrics(operatorMetrics.physicalWrittenBytes, operatorMetrics.numWrittenFiles) metrics("writeIONanos") += operatorMetrics.writeIOTime metrics("wallNanos") += operatorMetrics.wallNanos - metrics("numWrittenFiles") += operatorMetrics.numWrittenFiles metrics("loadLazyVectorTime") += operatorMetrics.loadLazyVectorTime } } diff --git a/pom.xml b/pom.xml index d69c773f64..aa8ea0754e 100644 --- a/pom.xml +++ b/pom.xml @@ -46,6 +46,7 @@ <modules> <module>gluten-core</module> <module>gluten-substrait</module> + <module>backends-core</module> <module>gluten-ui</module> <module>package</module> <module>shims</module> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
