This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 152ec52cb9 [GLUTEN-12597][CORE] Migrate Substrait extension
referencing from URI to URN (#12604)
152ec52cb9 is described below
commit 152ec52cb9b252b13d0f62c4eaaae384c8bf4450
Author: Niels Pardon <[email protected]>
AuthorDate: Mon Jul 27 16:02:04 2026 +0200
[GLUTEN-12597][CORE] Migrate Substrait extension referencing from URI to
URN (#12604)
Adopt the 0.98 URN-based simple-extension referencing model
(substrait-io/substrait#971):
- extensions.proto: SimpleExtensionURI -> SimpleExtensionURN;
extension_uri_anchor -> extension_urn_anchor; uri -> urn; and in
ExtensionType/ExtensionTypeVariation/ExtensionFunction, extension_uri_reference
(field 1) is reserved and replaced by extension_urn_reference (field 4).
- plan.proto / extended_expression.proto: extension_uris (field 1) is
reserved, replaced by extension_urns = 8 (repeated SimpleExtensionURN).
Velox producer (SubstraitExtensionCollector, VeloxToSubstraitPlan) now
emits extension_urns / extension_urn_reference. Gluten still maps every
function to one catch-all anchor and resolves by name, so the emitted URN is a
single placeholder in the required extension:<OWNER>:<ID> format
(extension:org.apache.gluten:functions) rather than an empty string; consuming
the upstream io.substrait function extensions is a follow-up. Gluten's JVM
producer never emitted extension_uris, so it nee [...]
AdvancedExtension.optimization (also repeated in 0.98) and the additive
Plan fields (parameter_bindings/type_aliases/execution_behavior) are handled in
follow-up increments. Part of #12597.
Validated locally: gluten-substrait JVM build + Velox native build
(libgluten/libvelox).
---
cpp/velox/substrait/SubstraitExtensionCollector.cc | 15 ++++---
cpp/velox/substrait/VeloxToSubstraitPlan.cc | 2 +-
cpp/velox/tests/data/q1_first_stage.json | 24 +++++------
cpp/velox/tests/data/q6_first_stage.json | 20 ++++-----
cpp/velox/tests/data/substrait_virtualTable.json | 2 +-
.../proto/substrait/extended_expression.proto | 9 ++--
.../proto/substrait/extensions/extensions.proto | 48 +++++++++++++---------
.../resources/substrait/proto/substrait/plan.proto | 8 ++--
8 files changed, 72 insertions(+), 56 deletions(-)
diff --git a/cpp/velox/substrait/SubstraitExtensionCollector.cc
b/cpp/velox/substrait/SubstraitExtensionCollector.cc
index 472ef04e3f..51695437c4 100644
--- a/cpp/velox/substrait/SubstraitExtensionCollector.cc
+++ b/cpp/velox/substrait/SubstraitExtensionCollector.cc
@@ -39,15 +39,18 @@ bool
SubstraitExtensionCollector::BiDirectionHashMap<T>::putIfAbsent(const int&
}
void SubstraitExtensionCollector::addExtensionsToPlan(::substrait::Plan* plan)
const {
- using SimpleExtensionURI = ::substrait::extensions::SimpleExtensionURI;
- // Currently we don't introduce any substrait extension YAML files, so always
- // only have one URI.
- SimpleExtensionURI* extensionUri = plan->add_extension_uris();
- extensionUri->set_extension_uri_anchor(1);
+ using SimpleExtensionURN = ::substrait::extensions::SimpleExtensionURN;
+ // Currently we don't map functions to their individual Substrait extension
+ // YAML files, so we emit a single catch-all URN and resolve functions by
+ // name. The URN follows the required extension:<OWNER>:<ID> format;
consuming
+ // the upstream io.substrait function extensions is left to a follow-up.
+ SimpleExtensionURN* extensionUrn = plan->add_extension_urns();
+ extensionUrn->set_extension_urn_anchor(1);
+ extensionUrn->set_urn("extension:org.apache.gluten:functions");
for (const auto& [referenceNum, functionId] :
extensionFunctions_->forwardMap()) {
auto extensionFunction =
plan->add_extensions()->mutable_extension_function();
-
extensionFunction->set_extension_uri_reference(extensionUri->extension_uri_anchor());
+
extensionFunction->set_extension_urn_reference(extensionUrn->extension_urn_anchor());
extensionFunction->set_function_anchor(referenceNum);
extensionFunction->set_name(functionId.signature);
}
diff --git a/cpp/velox/substrait/VeloxToSubstraitPlan.cc
b/cpp/velox/substrait/VeloxToSubstraitPlan.cc
index bc9132a67b..39d9d2e152 100644
--- a/cpp/velox/substrait/VeloxToSubstraitPlan.cc
+++ b/cpp/velox/substrait/VeloxToSubstraitPlan.cc
@@ -103,7 +103,7 @@ AggregateCompanion toAggregateCompanion(const
core::AggregationNode::Aggregate&
// Add unknown type in extension.
auto unknownType = substraitPlan->add_extensions()->mutable_extension_type();
- unknownType->set_extension_uri_reference(0);
+ unknownType->set_extension_urn_reference(0);
unknownType->set_type_anchor(0);
unknownType->set_name("UNKNOWN");
diff --git a/cpp/velox/tests/data/q1_first_stage.json
b/cpp/velox/tests/data/q1_first_stage.json
index 1b9ba06231..1413ffbd25 100644
--- a/cpp/velox/tests/data/q1_first_stage.json
+++ b/cpp/velox/tests/data/q1_first_stage.json
@@ -1,70 +1,70 @@
{
- "extension_uris": [
+ "extension_urns": [
{
- "extension_uri_anchor": 1,
- "uri": "/functions_datetime.yaml"
+ "extension_urn_anchor": 1,
+ "urn": "extension:io.substrait:functions_datetime"
}
],
"extensions": [
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 1,
"name": "lte:fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 6,
"name": "sum:opt_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 3,
"name": "subtract:opt_fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 9,
"name": "is_not_null:fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 2,
"name": "and:bool_bool"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 5,
"name": "add:opt_fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 7,
"name": "count:opt_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 4,
"name": "multiply:opt_fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 8,
"name": "count:opt_i32"
}
diff --git a/cpp/velox/tests/data/q6_first_stage.json
b/cpp/velox/tests/data/q6_first_stage.json
index 031793a2b1..7c4d12f1c9 100644
--- a/cpp/velox/tests/data/q6_first_stage.json
+++ b/cpp/velox/tests/data/q6_first_stage.json
@@ -1,56 +1,56 @@
{
- "extension_uris": [
+ "extension_urns": [
{
- "extension_uri_anchor": 1,
- "uri": "/functions_boolean.yaml"
+ "extension_urn_anchor": 1,
+ "urn": "extension:io.substrait:functions_boolean"
}
],
"extensions": [
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 4,
"name": "lte:fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 6,
"name": "sum:opt_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 3,
"name": "lt:fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 7,
"name": "is_not_null:fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 1,
"name": "and:bool_bool"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 2,
"name": "gte:fp64_fp64"
}
},
{
"extension_function": {
- "extension_uri_reference": 1,
+ "extension_urn_reference": 1,
"function_anchor": 5,
"name": "multiply:opt_fp64_fp64"
}
diff --git a/cpp/velox/tests/data/substrait_virtualTable.json
b/cpp/velox/tests/data/substrait_virtualTable.json
index 06536fadd7..5888df257e 100644
--- a/cpp/velox/tests/data/substrait_virtualTable.json
+++ b/cpp/velox/tests/data/substrait_virtualTable.json
@@ -1,5 +1,5 @@
{
- "extension_uris": [],
+ "extension_urns": [],
"extensions": [],
"relations": [
{
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto
index 5d11520559..60b7d5208d 100755
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/extended_expression.proto
@@ -9,7 +9,7 @@ import "substrait/plan.proto";
import "substrait/type.proto";
option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto";
+option go_package =
"github.com/substrait-io/substrait-protobuf/go/substraitpb";
option java_multiple_files = true;
option java_package = "io.substrait.proto";
@@ -25,12 +25,15 @@ message ExpressionReference {
// Describe a set of operations to complete.
// For compactness sake, identifiers are normalized at the plan level.
message ExtendedExpression {
+ reserved 1;
+
// Substrait version of the expression. Optional up to 0.17.0, required for
later
// versions.
Version version = 7;
- // a list of yaml specifications this expression may depend on
- repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
+ // a list of extension specifications this expression may depend on,
+ // referenced by Extension URN
+ repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
// a list of extensions this expression may depend on
repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto
index d3af5f3d9e..1e24ace044 100644
---
a/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto
+++
b/gluten-substrait/src/main/resources/substrait/proto/substrait/extensions/extensions.proto
@@ -6,21 +6,22 @@ package substrait.extensions;
import "google/protobuf/any.proto";
option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto/extensions";
+option go_package =
"github.com/substrait-io/substrait-protobuf/go/substraitpb/extensions";
option java_multiple_files = true;
option java_package = "io.substrait.proto";
-message SimpleExtensionURI {
+message SimpleExtensionURN {
// A surrogate key used in the context of a single plan used to reference the
- // URI associated with an extension.
- uint32 extension_uri_anchor = 1;
+ // URN associated with an extension.
+ // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics.
+ uint32 extension_urn_anchor = 1;
- // The URI where this extension YAML can be retrieved. This is the
"namespace"
- // of this extension.
- string uri = 2;
+ // The extension URN that uniquely identifies this extension. This must
follow the
+ // format extension:<OWNER>:<ID> and serves as the "namespace" of this
extension.
+ string urn = 2;
}
-// Describes a mapping between a specific extension entity and the uri where
+// Describes a mapping between a specific extension entity and the URN where
// that extension can be found.
message SimpleExtensionDeclaration {
oneof mapping_type {
@@ -31,11 +32,14 @@ message SimpleExtensionDeclaration {
// Describes a Type
message ExtensionType {
- // references the extension_uri_anchor defined for a specific extension
URI.
- uint32 extension_uri_reference = 1;
+ reserved 1;
+
+ // references the extension_urn_anchor defined for a specific extension
URN.
+ uint32 extension_urn_reference = 4;
// A surrogate key used in the context of a single plan to reference a
- // specific extension type
+ // specific extension type.
+ // 0 is a valid anchor/reference, but prefer non-zero values for
ergonomics.
uint32 type_anchor = 2;
// the name of the type in the defined extension YAML.
@@ -43,11 +47,14 @@ message SimpleExtensionDeclaration {
}
message ExtensionTypeVariation {
- // references the extension_uri_anchor defined for a specific extension
URI.
- uint32 extension_uri_reference = 1;
+ reserved 1;
+
+ // references the extension_urn_anchor defined for a specific extension
URN.
+ uint32 extension_urn_reference = 4;
// A surrogate key used in the context of a single plan to reference a
- // specific type variation
+ // specific type variation.
+ // Use non-zero values; 0 is reserved for the system-preferred variation.
uint32 type_variation_anchor = 2;
// the name of the type in the defined extension YAML.
@@ -55,16 +62,17 @@ message SimpleExtensionDeclaration {
}
message ExtensionFunction {
- // references the extension_uri_anchor defined for a specific extension
URI.
- uint32 extension_uri_reference = 1;
+ reserved 1;
+
+ // references the extension_urn_anchor defined for a specific extension
URN.
+ uint32 extension_urn_reference = 4;
// A surrogate key used in the context of a single plan to reference a
- // specific function
+ // specific function.
+ // 0 is a valid anchor/reference, but prefer non-zero values for
ergonomics.
uint32 function_anchor = 2;
- // A simple name if there is only one impl for the function within the
YAML.
- // A compound name, referencing that includes type short names if there is
- // more than one impl per name in the YAML.
+ // A function signature
string name = 3;
}
}
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto
index e5657fb8f1..2fddc1ddec 100644
--- a/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto
+++ b/gluten-substrait/src/main/resources/substrait/proto/substrait/plan.proto
@@ -7,7 +7,7 @@ import "substrait/algebra.proto";
import "substrait/extensions/extensions.proto";
option csharp_namespace = "Substrait.Protobuf";
-option go_package = "github.com/substrait-io/substrait-go/proto";
+option go_package =
"github.com/substrait-io/substrait-protobuf/go/substraitpb";
option java_multiple_files = true;
option java_package = "io.substrait.proto";
@@ -24,12 +24,14 @@ message PlanRel {
// Describe a set of operations to complete.
// For compactness sake, identifiers are normalized at the plan level.
message Plan {
+ reserved 1;
+
// Substrait version of the plan. Optional up to 0.17.0, required for later
// versions.
Version version = 6;
- // a list of yaml specifications this plan may depend on
- repeated substrait.extensions.SimpleExtensionURI extension_uris = 1;
+ // a list of extension URNs this plan may depend on
+ repeated substrait.extensions.SimpleExtensionURN extension_urns = 8;
// a list of extensions this plan may depend on
repeated substrait.extensions.SimpleExtensionDeclaration extensions = 2;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
