This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e9c943e244 [GLUTEN-12597][CORE] Migrate Substrait type system to 0.98
temporal types (#12650)
e9c943e244 is described below
commit e9c943e2448fef890811fa44ff5e66481e77c5f1
Author: Niels Pardon <[email protected]>
AuthorDate: Wed Aug 19 15:25:02 2026 +0200
[GLUTEN-12597][CORE] Migrate Substrait type system to 0.98 temporal types
(#12650)
---
.../local-engine/Builder/SerializedPlanBuilder.cpp | 8 ++-
cpp-ch/local-engine/Parser/TypeParser.cpp | 4 +-
cpp/velox/substrait/SubstraitParser.cc | 4 +-
cpp/velox/substrait/VeloxToSubstraitType.cc | 7 ++-
.../substrait/type/TimestampNTZTypeNode.java | 6 +-
.../gluten/substrait/type/TimestampTypeNode.java | 6 +-
.../resources/substrait/proto/substrait/type.proto | 59 +++++++++++--------
.../apache/gluten/expression/ConverterUtils.scala | 8 +--
.../gluten/expression/ConverterUtilsSuite.scala | 67 ++++++++++++++++++++++
9 files changed, 128 insertions(+), 41 deletions(-)
diff --git a/cpp-ch/local-engine/Builder/SerializedPlanBuilder.cpp
b/cpp-ch/local-engine/Builder/SerializedPlanBuilder.cpp
index 3d80670087..41828b74e8 100644
--- a/cpp-ch/local-engine/Builder/SerializedPlanBuilder.cpp
+++ b/cpp-ch/local-engine/Builder/SerializedPlanBuilder.cpp
@@ -107,7 +107,9 @@ SchemaPtr SerializedSchemaBuilder::build()
else if (type == "Timestamp")
{
auto * t = type_struct->mutable_types()->Add();
- t->mutable_timestamp_tz()->set_nullability(
+ // CH DateTime64(6) is microsecond precision.
+ t->mutable_precision_timestamp_tz()->set_precision(6);
+ t->mutable_precision_timestamp_tz()->set_nullability(
this->nullability_map[name] ?
substrait::Type_Nullability_NULLABILITY_NULLABLE
:
substrait::Type_Nullability_NULLABILITY_REQUIRED);
}
@@ -256,7 +258,9 @@ std::shared_ptr<substrait::Type>
SerializedPlanBuilder::buildType(const DB::Data
const auto * ch_type_datetime64 =
checkAndGetDataType<DataTypeDateTime64>(ch_type_without_nullable.get());
if (ch_type_datetime64->getScale() != 6)
throw Exception(ErrorCodes::UNKNOWN_TYPE, "Spark doesn't support
converting from {}", ch_type->getName());
- res->mutable_timestamp_tz()->set_nullability(type_nullability);
+ // CH DateTime64(6) is microsecond precision.
+ res->mutable_precision_timestamp_tz()->set_precision(6);
+
res->mutable_precision_timestamp_tz()->set_nullability(type_nullability);
}
else if (which.isDate32())
res->mutable_date()->set_nullability(type_nullability);
diff --git a/cpp-ch/local-engine/Parser/TypeParser.cpp
b/cpp-ch/local-engine/Parser/TypeParser.cpp
index bdb8c52e9c..633cb3f93a 100644
--- a/cpp-ch/local-engine/Parser/TypeParser.cpp
+++ b/cpp-ch/local-engine/Parser/TypeParser.cpp
@@ -153,10 +153,10 @@ DB::DataTypePtr TypeParser::parseType(const
substrait::Type & substrait_type, st
ch_type = std::make_shared<DB::DataTypeFloat64>();
ch_type = tryWrapNullable(substrait_type.fp64().nullability(),
ch_type);
}
- else if (substrait_type.has_timestamp_tz())
+ else if (substrait_type.has_precision_timestamp_tz())
{
ch_type = std::make_shared<DB::DataTypeDateTime64>(6);
- ch_type = tryWrapNullable(substrait_type.timestamp_tz().nullability(),
ch_type);
+ ch_type =
tryWrapNullable(substrait_type.precision_timestamp_tz().nullability(), ch_type);
}
else if (substrait_type.has_date())
{
diff --git a/cpp/velox/substrait/SubstraitParser.cc
b/cpp/velox/substrait/SubstraitParser.cc
index 54bf8d4f24..a57b3f69cc 100644
--- a/cpp/velox/substrait/SubstraitParser.cc
+++ b/cpp/velox/substrait/SubstraitParser.cc
@@ -75,9 +75,9 @@ TypePtr SubstraitParser::parseType(const ::substrait::Type&
substraitType, bool
return UNKNOWN();
case ::substrait::Type::KindCase::kDate:
return DATE();
- case ::substrait::Type::KindCase::kTimestampTz:
+ case ::substrait::Type::KindCase::kPrecisionTimestampTz:
return TIMESTAMP();
- case ::substrait::Type::KindCase::kTimestamp:
+ case ::substrait::Type::KindCase::kPrecisionTimestamp:
return TIMESTAMP_UTC();
case ::substrait::Type::KindCase::kDecimal: {
auto precision = substraitType.decimal().precision();
diff --git a/cpp/velox/substrait/VeloxToSubstraitType.cc
b/cpp/velox/substrait/VeloxToSubstraitType.cc
index b6bcf3bcc9..c3f2d4b44d 100644
--- a/cpp/velox/substrait/VeloxToSubstraitType.cc
+++ b/cpp/velox/substrait/VeloxToSubstraitType.cc
@@ -88,9 +88,12 @@ const ::substrait::Type&
VeloxToSubstraitTypeConvertor::toSubstraitType(
break;
}
case velox::TypeKind::TIMESTAMP: {
- auto substraitTimestampTZ =
google::protobuf::Arena::CreateMessage<::substrait::Type_TimestampTZ>(&arena);
+ auto substraitTimestampTZ =
+
google::protobuf::Arena::CreateMessage<::substrait::Type_PrecisionTimestampTZ>(&arena);
+ // Velox TIMESTAMP is microsecond precision.
+ substraitTimestampTZ->set_precision(6);
substraitTimestampTZ->set_nullability(::substrait::Type_Nullability_NULLABILITY_NULLABLE);
- substraitType->set_allocated_timestamp_tz(substraitTimestampTZ);
+
substraitType->set_allocated_precision_timestamp_tz(substraitTimestampTZ);
break;
}
case velox::TypeKind::ARRAY: {
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampNTZTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampNTZTypeNode.java
index 83d27cfb09..bbf73ff02f 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampNTZTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampNTZTypeNode.java
@@ -26,7 +26,9 @@ public class TimestampNTZTypeNode extends TypeNode {
@Override
public Type toProtobuf() {
- Type.Timestamp.Builder timestampBuilder = Type.Timestamp.newBuilder();
+ Type.PrecisionTimestamp.Builder timestampBuilder =
Type.PrecisionTimestamp.newBuilder();
+ // Spark's TimestampNTZType is microsecond precision.
+ timestampBuilder.setPrecision(6);
if (nullable) {
timestampBuilder.setNullability(Type.Nullability.NULLABILITY_NULLABLE);
} else {
@@ -34,7 +36,7 @@ public class TimestampNTZTypeNode extends TypeNode {
}
Type.Builder builder = Type.newBuilder();
- builder.setTimestamp(timestampBuilder.build());
+ builder.setPrecisionTimestamp(timestampBuilder.build());
return builder.build();
}
}
diff --git
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
index 472df5da97..e30830399f 100644
---
a/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
+++
b/gluten-substrait/src/main/java/org/apache/gluten/substrait/type/TimestampTypeNode.java
@@ -26,7 +26,9 @@ public class TimestampTypeNode extends TypeNode {
@Override
public Type toProtobuf() {
- Type.TimestampTZ.Builder timestampBuilder = Type.TimestampTZ.newBuilder();
+ Type.PrecisionTimestampTZ.Builder timestampBuilder =
Type.PrecisionTimestampTZ.newBuilder();
+ // Spark's TimestampType is microsecond precision.
+ timestampBuilder.setPrecision(6);
if (nullable) {
timestampBuilder.setNullability(Type.Nullability.NULLABILITY_NULLABLE);
} else {
@@ -34,7 +36,7 @@ public class TimestampTypeNode extends TypeNode {
}
Type.Builder builder = Type.newBuilder();
- builder.setTimestampTz(timestampBuilder.build());
+ builder.setPrecisionTimestampTz(timestampBuilder.build());
return builder.build();
}
}
diff --git
a/gluten-substrait/src/main/resources/substrait/proto/substrait/type.proto
b/gluten-substrait/src/main/resources/substrait/proto/substrait/type.proto
index b5fcb95623..8f226aa5b8 100644
--- a/gluten-substrait/src/main/resources/substrait/proto/substrait/type.proto
+++ b/gluten-substrait/src/main/resources/substrait/proto/substrait/type.proto
@@ -11,6 +11,11 @@ option java_multiple_files = true;
option java_package = "io.substrait.proto";
message Type {
+ // 14/17/29 held the pre-0.98 Timestamp/Time/TimestampTZ types (now
+ // PrecisionTimestamp/PrecisionTime/PrecisionTimestampTZ); 31 held the
+ // deprecated user_defined_type_reference.
+ reserved 14, 17, 29, 31;
+
oneof kind {
Boolean bool = 1;
I8 i8 = 2;
@@ -21,18 +26,18 @@ message Type {
FP64 fp64 = 11;
String string = 12;
Binary binary = 13;
- Timestamp timestamp = 14;
Date date = 16;
- Time time = 17;
IntervalYear interval_year = 19;
IntervalDay interval_day = 20;
- TimestampTZ timestamp_tz = 29;
UUID uuid = 32;
FixedChar fixed_char = 21;
VarChar varchar = 22;
FixedBinary fixed_binary = 23;
Decimal decimal = 24;
+ PrecisionTime precision_time = 36;
+ PrecisionTimestamp precision_timestamp = 33;
+ PrecisionTimestampTZ precision_timestamp_tz = 34; // value is since UNIX
epoch in UTC
Struct struct = 25;
List list = 27;
@@ -40,13 +45,11 @@ message Type {
UserDefined user_defined = 30;
- // Deprecated in favor of user_defined, which allows nullability and
- // variations to be specified. If user_defined_type_reference is
- // encountered, treat it as being non-nullable and having the default
- // variation.
- uint32 user_defined_type_reference = 31 [deprecated = true];
-
- Nothing nothing = 33;
+ // Gluten fork: a bottom type used to carry Spark's NullType. Relocated off
+ // field 33 (which is precision_timestamp in Substrait 0.98). Pending
+ // removal in a follow-up that maps NullType to a typed null (i32); see the
+ // Substrait consolidation plan.
+ Nothing nothing = 40;
}
enum Nullability {
@@ -104,26 +107,11 @@ message Type {
Nullability nullability = 2;
}
- message Timestamp {
- uint32 type_variation_reference = 1;
- Nullability nullability = 2;
- }
-
message Date {
uint32 type_variation_reference = 1;
Nullability nullability = 2;
}
- message Time {
- uint32 type_variation_reference = 1;
- Nullability nullability = 2;
- }
-
- message TimestampTZ {
- uint32 type_variation_reference = 1;
- Nullability nullability = 2;
- }
-
message IntervalYear {
uint32 type_variation_reference = 1;
Nullability nullability = 2;
@@ -165,6 +153,27 @@ message Type {
Nullability nullability = 4;
}
+ message PrecisionTime {
+ // Sub-second precision, 0 means the value given is in seconds, 3 is
milliseconds, 6 microseconds, 9 is nanoseconds, 12 is picoseconds
+ int32 precision = 1;
+ uint32 type_variation_reference = 2;
+ Nullability nullability = 3;
+ }
+
+ message PrecisionTimestamp {
+ // Sub-second precision, 0 means the value given is in seconds, 3 is
milliseconds, 6 microseconds, 9 is nanoseconds, 12 is picoseconds
+ int32 precision = 1;
+ uint32 type_variation_reference = 2;
+ Nullability nullability = 3;
+ }
+
+ message PrecisionTimestampTZ {
+ // Sub-second precision, 0 means the value given is in seconds, 3 is
milliseconds, 6 microseconds, 9 is nanoseconds, 12 is picoseconds
+ int32 precision = 1;
+ uint32 type_variation_reference = 2;
+ Nullability nullability = 3;
+ }
+
message Struct {
repeated Type types = 1;
uint32 type_variation_reference = 2;
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
index 53fa2280d6..ca83ccbd5b 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
@@ -160,7 +160,7 @@ object ConverterUtils extends Logging {
(StringType, isNullable(substraitType.getString.getNullability))
case Type.KindCase.BINARY =>
(BinaryType, isNullable(substraitType.getBinary.getNullability))
- case Type.KindCase.TIMESTAMP =>
+ case Type.KindCase.PRECISION_TIMESTAMP =>
try {
(
Class
@@ -168,13 +168,13 @@ object ConverterUtils extends Logging {
.getField("MODULE$")
.get(null)
.asInstanceOf[DataType],
- isNullable(substraitType.getTimestamp.getNullability))
+ isNullable(substraitType.getPrecisionTimestamp.getNullability))
} catch {
case _: ReflectiveOperationException =>
throw new GlutenNotSupportException(s"Type $substraitType not
supported.")
}
- case Type.KindCase.TIMESTAMP_TZ =>
- (TimestampType,
isNullable(substraitType.getTimestampTz.getNullability))
+ case Type.KindCase.PRECISION_TIMESTAMP_TZ =>
+ (TimestampType,
isNullable(substraitType.getPrecisionTimestampTz.getNullability))
case Type.KindCase.DATE =>
(DateType, isNullable(substraitType.getDate.getNullability))
case Type.KindCase.DECIMAL =>
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/expression/ConverterUtilsSuite.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/expression/ConverterUtilsSuite.scala
new file mode 100644
index 0000000000..268538af15
--- /dev/null
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/expression/ConverterUtilsSuite.scala
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.gluten.expression
+
+import org.apache.spark.sql.types.{DataType, TimestampType}
+
+import io.substrait.proto.Type
+import org.scalatest.funsuite.AnyFunSuiteLike
+
+/**
+ * Guards the Substrait 0.98 temporal type migration: Spark's TimestampType
maps to
+ * `PrecisionTimestampTZ` and TimestampNTZType maps to `PrecisionTimestamp`,
both at microsecond
+ * precision (6), and both round-trip back to the original Spark type.
+ */
+class ConverterUtilsSuite extends AnyFunSuiteLike {
+
+ test("TimestampType emits PrecisionTimestampTZ(precision=6) and
round-trips") {
+ Seq(true, false).foreach {
+ nullable =>
+ val proto = ConverterUtils.getTypeNode(TimestampType,
nullable).toProtobuf
+ assert(proto.getKindCase === Type.KindCase.PRECISION_TIMESTAMP_TZ)
+ assert(proto.getPrecisionTimestampTz.getPrecision === 6)
+
assert(ConverterUtils.isNullable(proto.getPrecisionTimestampTz.getNullability)
=== nullable)
+
+ val (dataType, parsedNullable) =
ConverterUtils.parseFromSubstraitType(proto)
+ assert(dataType === TimestampType)
+ assert(parsedNullable === nullable)
+ }
+ }
+
+ test("TimestampNTZType emits PrecisionTimestamp(precision=6) and
round-trips") {
+ // TimestampNTZType is package-private before Spark 3.4, so resolve the
singleton
+ // reflectively (mirroring ConverterUtils.parseFromSubstraitType) to keep
this suite
+ // compilable across all supported Spark versions.
+ val timestampNTZType = Class
+ .forName("org.apache.spark.sql.types.TimestampNTZType$")
+ .getField("MODULE$")
+ .get(null)
+ .asInstanceOf[DataType]
+
+ Seq(true, false).foreach {
+ nullable =>
+ val proto = ConverterUtils.getTypeNode(timestampNTZType,
nullable).toProtobuf
+ assert(proto.getKindCase === Type.KindCase.PRECISION_TIMESTAMP)
+ assert(proto.getPrecisionTimestamp.getPrecision === 6)
+
assert(ConverterUtils.isNullable(proto.getPrecisionTimestamp.getNullability)
=== nullable)
+
+ val (dataType, parsedNullable) =
ConverterUtils.parseFromSubstraitType(proto)
+ assert(dataType === timestampNTZType)
+ assert(parsedNullable === nullable)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
