This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fb588bc4c9 [GLUTEN-12597][CORE] Migrate Rel oneof to Substrait 0.98
(park GenerateRel/WindowGroupLimitRel) (#13210)
fb588bc4c9 is described below
commit fb588bc4c965db7895e3fc2c781a62a9fb6cbd01
Author: Niels Pardon <[email protected]>
AuthorDate: Fri Oct 23 14:50:16 2026 +0200
[GLUTEN-12597][CORE] Migrate Rel oneof to Substrait 0.98 (park
GenerateRel/WindowGroupLimitRel) (#13210)
---
.../local-engine/Parser/SerializedPlanParser.cpp | 10 +++
docs/developers/SubstraitModifications.md | 5 ++
.../substrait/proto/substrait/algebra.proto | 96 ++++++++++++++++++---
.../gluten/substrait/rel/RelProtoSuite.scala | 99 ++++++++++++++++++++++
4 files changed, 198 insertions(+), 12 deletions(-)
diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
index 594484adae..556e24d0ad 100644
--- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
+++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
@@ -52,6 +52,7 @@
#include <Storages/SubstraitSource/SubstraitFileSource.h>
#include <Storages/SubstraitSource/SubstraitFileSourceStep.h>
#include <google/protobuf/wrappers.pb.h>
+#include <magic_enum.hpp>
#include <Common/BlockTypeUtils.h>
#include <Common/DebugUtils.h>
#include <Common/Exception.h>
@@ -60,6 +61,15 @@
#include <Common/logger_useful.h>
#include <Common/typeid_cast.h>
+/// Gluten's own relations sit at 1000+ in the Rel oneof, past magic_enum's
default reflection range,
+/// so widen it to keep their metric names (e.g. kGenerate, kWindowGroupLimit).
+template <>
+struct magic_enum::customize::enum_range<substrait::Rel::RelTypeCase>
+{
+ static constexpr int min = 0;
+ static constexpr int max = 1001;
+};
+
namespace DB
{
namespace Setting
diff --git a/docs/developers/SubstraitModifications.md
b/docs/developers/SubstraitModifications.md
index 3299c5f198..e99cd818a2 100644
--- a/docs/developers/SubstraitModifications.md
+++ b/docs/developers/SubstraitModifications.md
@@ -33,6 +33,11 @@ changed `Unbounded` in `WindowFunction` into
`Unbounded_Preceding` and `Unbounde
* Added `output_schema` in
RelRoot([#1901](https://github.com/apache/gluten/pull/1901)).
* Added `ExpandRel`([#1361](https://github.com/apache/gluten/pull/1361)).
* Added `GenerateRel`([#574](https://github.com/apache/gluten/pull/574)).
+* Rebased the `Rel` oneof onto upstream `v0.98.0`: every upstream relation now
sits at its upstream number
+(`exchange` 15, `expand` 16, `window` 17, `ddl` 20, `reference` 21, `update`
22, `lateral_join` 24), with
+`LateralJoinRel`, `UpdateRel`, the top-level `NamedTable` and
`DdlRel.common`/`advanced_extension` vendored
+verbatim, and `ReferenceRel` moved out of `AggregateFunction` to the top
level. Gluten's `GenerateRel` and
+`WindowGroupLimitRel` moved to 1000 and 1001.
* Added `PartitionColumn` in
`LocalFiles`([#2405](https://github.com/apache/gluten/pull/2405)).
* Added `WriteRel` ([#3690](https://github.com/apache/gluten/pull/3690)).
* Added `TopNRel` ([#5409](https://github.com/apache/gluten/pull/5409)).
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
index a698b297b6..562593ba91 100644
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/algebra.proto
@@ -346,6 +346,40 @@ message JoinRel {
substrait.extensions.AdvancedExtension advanced_extension = 10;
}
+// The binary lateral JOIN relational operator left-join-right. This is
+// semantically identical to JoinRel, except the right input is evaluated once
+// per row of the left input and may reference fields from the current left row
+// via OuterReference rel_reference.
+//
+// LateralJoinRel must set RelCommon.rel_anchor so the right input can
+// reference fields of the current left row.
+message LateralJoinRel {
+ RelCommon common = 1;
+ // The independent input whose rows drive evaluation of the lateral join.
+ Rel left = 2;
+
+ // The dependent input evaluated once per left row. It may reference fields
+ // from the current left row using OuterReference.rel_reference pointing to
+ // this LateralJoinRel's RelCommon.rel_anchor.
+ Rel right = 3;
+ // A boolean condition evaluated over the left and right inputs that
+ // determines whether a pair of records is a match.
+ Expression expression = 4;
+ // An optional boolean filter applied to each output record after
+ // join-type-specific output formation is complete.
+ // Semantically equivalent to placing a FilterRel directly above this join.
+ Expression post_join_filter = 5;
+
+ // Only INNER and left-oriented join types are valid for lateral joins:
+ // INNER, LEFT, LEFT_SEMI, LEFT_ANTI, LEFT_SINGLE, LEFT_MARK.
+ // RIGHT-oriented join types and OUTER are not valid because the right input
+ // is evaluated in the context of each left row and has no independent
+ // existence.
+ JoinRel.JoinType type = 6;
+
+ substrait.extensions.AdvancedExtension advanced_extension = 10;
+}
+
// Cartesian product relational operator of two tables (left and right)
message CrossRel {
RelCommon common = 1;
@@ -691,22 +725,29 @@ message Rel {
AggregateRel aggregate = 4;
SortRel sort = 5;
JoinRel join = 6;
+ LateralJoinRel lateral_join = 24;
ProjectRel project = 7;
SetRel set = 8;
ExtensionSingleRel extension_single = 9;
ExtensionMultiRel extension_multi = 10;
ExtensionLeafRel extension_leaf = 11;
CrossRel cross = 12;
- //Physical relations
+ ReferenceRel reference = 21;
+ WriteRel write = 19;
+ DdlRel ddl = 20;
+ UpdateRel update = 22;
+ // Physical relations
HashJoinRel hash_join = 13;
MergeJoinRel merge_join = 14;
- ExpandRel expand = 15;
- ConsistentPartitionWindowRel window = 16;
- GenerateRel generate = 17;
- WriteRel write = 19;
- TopNRel top_n = 23;
- WindowGroupLimitRel windowGroupLimit = 20;
NestedLoopJoinRel nested_loop_join = 18;
+ ConsistentPartitionWindowRel window = 17;
+ ExchangeRel exchange = 15;
+ ExpandRel expand = 16;
+ TopNRel top_n = 23;
+ // Gluten additions, numbered from 1000 so future upstream relations
+ // cannot collide with them.
+ GenerateRel generate = 1000;
+ WindowGroupLimitRel windowGroupLimit = 1001;
}
}
@@ -747,6 +788,7 @@ message DdlRel {
// The body of the CREATE VIEW
Rel view_definition = 7;
+ RelCommon common = 8;
enum DdlObject {
DDL_OBJECT_UNSPECIFIED = 0;
@@ -769,6 +811,9 @@ message DdlRel {
// An operation that removes an object from the system (without throwing
an exception if the object did not exist)
DDL_OP_DROP_IF_EXIST = 5;
}
+
+ substrait.extensions.AdvancedExtension advanced_extension = 9;
+
//TODO add PK/constraints/indexes/etc..?
}
@@ -853,6 +898,33 @@ message WriteRel {
}
}
+// The operator that modifies the columns of a table
+message UpdateRel {
+ oneof update_type {
+ NamedTable named_table = 1;
+ }
+
+ NamedStruct table_schema = 2; // The full schema of the named_table
+ Expression condition = 3; // condition to be met for the update to be
applied on a record
+
+ // The list of transformations to apply to the columns of the named_table
+ repeated TransformExpression transformations = 4;
+
+ message TransformExpression {
+ Expression transformation = 1; // the transformation to apply
+ int32 column_target = 2; // index of the column to apply the
transformation to
+ }
+
+ substrait.extensions.AdvancedExtension advanced_extension = 5;
+}
+
+// A base table. The list of string is used to represent namespacing (e.g.,
mydb.mytable).
+// This assumes shared catalog between systems exchanging a message.
+message NamedTable {
+ repeated string names = 1;
+ substrait.extensions.AdvancedExtension advanced_extension = 10;
+}
+
// The hash equijoin join operator will build a hash table out of the right
input based on a set of join keys.
// It will then probe that hash table for incoming inputs, finding matches.
message HashJoinRel {
@@ -1681,10 +1753,10 @@ message AggregateFunction {
// Use only distinct values in the aggregation calculation.
AGGREGATION_INVOCATION_DISTINCT = 2;
}
+}
- // This rel is used to create references,
- // in case we refer to a RelRoot field names will be ignored
- message ReferenceRel {
- int32 subtree_ordinal = 1;
- }
+// This rel is used to create references,
+// in case we refer to a RelRoot field names will be ignored
+message ReferenceRel {
+ int32 subtree_ordinal = 1;
}
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/substrait/rel/RelProtoSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/substrait/rel/RelProtoSuite.scala
new file mode 100644
index 0000000000..ff8e172163
--- /dev/null
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/substrait/rel/RelProtoSuite.scala
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.substrait.rel
+
+import com.google.protobuf.Descriptors.Descriptor
+import io.substrait.proto.{AggregateFunction, DdlRel, Rel}
+import org.scalatest.funsuite.AnyFunSuite
+
+import scala.collection.JavaConverters._
+
+/**
+ * Pins the wire tags of the vendored `Rel.rel_type` oneof. Producer and
consumer share one schema,
+ * so a renumber round-trips cleanly through the generated classes and cannot
be caught by
+ * exercising them; these assert on the descriptors instead.
+ *
+ * Gluten's own relations live in the 1000 graft range so that relations added
by upstream cannot
+ * collide with them.
+ */
+class RelProtoSuite extends AnyFunSuite {
+
+ private def assertFieldNumbers(descriptor: Descriptor, expected: (String,
Int)*): Unit =
+ expected.foreach {
+ case (name, number) =>
+ val field = descriptor.findFieldByName(name)
+ assert(field != null, s"${descriptor.getName} has no field named
$name")
+ assert(field.getNumber === number, s"${descriptor.getName} field $name
changed its number")
+ }
+
+ private val standardRelations = Seq(
+ "read" -> 1,
+ "filter" -> 2,
+ "fetch" -> 3,
+ "aggregate" -> 4,
+ "sort" -> 5,
+ "join" -> 6,
+ "project" -> 7,
+ "set" -> 8,
+ "extension_single" -> 9,
+ "extension_multi" -> 10,
+ "extension_leaf" -> 11,
+ "cross" -> 12,
+ "hash_join" -> 13,
+ "merge_join" -> 14,
+ "exchange" -> 15,
+ "expand" -> 16,
+ "window" -> 17,
+ "nested_loop_join" -> 18,
+ "write" -> 19,
+ "ddl" -> 20,
+ "reference" -> 21,
+ "update" -> 22,
+ "top_n" -> 23,
+ "lateral_join" -> 24
+ )
+
+ private val glutenRelations = Seq("generate" -> 1000, "windowGroupLimit" ->
1001)
+
+ test("the standard relations carry their expected tags") {
+ assertFieldNumbers(Rel.getDescriptor, standardRelations: _*)
+ }
+
+ test("Gluten-local relations live in the 1000 graft range") {
+ assertFieldNumbers(Rel.getDescriptor, glutenRelations: _*)
+ }
+
+ test("rel_type holds exactly the standard and Gluten-local relations") {
+ val rel = Rel.getDescriptor
+ val oneof = rel.getOneofs.asScala.find(_.getName == "rel_type")
+ assert(oneof.isDefined, "Rel has no rel_type oneof")
+ val actual = oneof.get.getFields.asScala.map(f => f.getName ->
f.getNumber).toSet
+ assert(actual === (standardRelations ++ glutenRelations).toSet)
+ }
+
+ test("ReferenceRel is a top-level message") {
+ val reference = Rel.getDescriptor.findFieldByName("reference")
+ assert(reference.getMessageType.getFullName === "substrait.ReferenceRel")
+ assert(
+ AggregateFunction.getDescriptor.findNestedTypeByName("ReferenceRel") ===
null,
+ "ReferenceRel must not be nested inside AggregateFunction")
+ }
+
+ test("DdlRel carries common and advanced_extension") {
+ assertFieldNumbers(DdlRel.getDescriptor, "common" -> 8,
"advanced_extension" -> 9)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
