This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch branch-0.6
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/branch-0.6 by this push:
new 8b55591ca [#4743]fix(trino-connector): Fix the default precision of
time and timestamp in the Iceberg catalog. (#4936)
8b55591ca is described below
commit 8b55591ca873d8c84897f83b747aad1bcc5d97fb
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Sep 14 20:56:21 2024 +0800
[#4743]fix(trino-connector): Fix the default precision of time and
timestamp in the Iceberg catalog. (#4936)
### What changes were proposed in this pull request?
Fix the default precision of time and timestamp in the Iceberg catalog.
It causes Trino to be unable to read Iceberg tables with data of time
and timestamp
### Why are the changes needed?
Fix: #4743
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
New UT, IT
Co-authored-by: Yuhui <[email protected]>
---
.../testsets/lakehouse-iceberg/00006_datatype.sql | 2 +-
.../testsets/lakehouse-iceberg/00006_datatype.txt | 10 +++++-----
.../catalog/iceberg/IcebergDataTypeTransformer.java | 14 +++++++++++++-
.../trino/connector/util/GeneralDataTypeTransformer.java | 11 +++++++++--
.../catalog/iceberg/TestIcebergDataTypeTransformer.java | 14 ++++++++++++++
.../trino/connector/util/TestDataTypeTransformer.java | 7 +++++++
6 files changed, 49 insertions(+), 9 deletions(-)
diff --git
a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql
b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql
index 728545888..45438c927 100644
---
a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql
+++
b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.sql
@@ -2,7 +2,7 @@ CREATE SCHEMA gt_iceberg.gt_db2;
USE gt_iceberg.gt_db2;
--- Unsupported Type: TINYINT, SMALLINT
+-- Unsupported Type: CHAR TINYINT, SMALLINT
CREATE TABLE tb01 (
f1 VARCHAR,
f3 VARBINARY,
diff --git
a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
index ee97371a6..b49c93c2e 100644
---
a/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
+++
b/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
@@ -15,8 +15,8 @@ CREATE TABLE
f11 integer,
f12 bigint,
f13 date,
- f14 time(3),
- f15 timestamp(3)
+ f14 time(6),
+ f15 timestamp(6)
)
COMMENT ''
WITH (
@@ -28,7 +28,7 @@ INSERT: 1 row
INSERT: 1 row
-"Sample text
1","65","123.456","7.89","12.34","true","1000","1000","100000","2024-01-01","08:00:00.000","2024-01-01
08:00:00.000"
+"Sample text
1","65","123.456","7.89","12.34","true","1000","1000","100000","2024-01-01","08:00:00.000000","2024-01-01
08:00:00.000000"
"","","","","","","","","","","",""
CREATE TABLE
@@ -44,8 +44,8 @@ CREATE TABLE
f11 integer NOT NULL,
f12 bigint NOT NULL,
f13 date NOT NULL,
- f14 time(3) NOT NULL,
- f15 timestamp(3) NOT NULL
+ f14 time(6) NOT NULL,
+ f15 timestamp(6) NOT NULL
)
COMMENT ''
WITH (
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java
index 35cecd1a4..2c4fa12f8 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergDataTypeTransformer.java
@@ -20,6 +20,9 @@
package org.apache.gravitino.trino.connector.catalog.iceberg;
import io.trino.spi.TrinoException;
+import io.trino.spi.type.TimeType;
+import io.trino.spi.type.TimestampType;
+import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.VarbinaryType;
import io.trino.spi.type.VarcharType;
import org.apache.gravitino.rel.types.Type;
@@ -45,7 +48,6 @@ public class IcebergDataTypeTransformer extends
GeneralDataTypeTransformer {
GravitinoErrorCode.GRAVITINO_ILLEGAL_ARGUMENT,
"Iceberg does not support the datatype VARCHAR with length");
}
-
return Types.StringType.get();
}
@@ -56,7 +58,17 @@ public class IcebergDataTypeTransformer extends
GeneralDataTypeTransformer {
public io.trino.spi.type.Type getTrinoType(Type type) {
if (Name.FIXED == type.name()) {
return VarbinaryType.VARBINARY;
+ } else if (Name.TIME == type.name()) {
+ return TimeType.TIME_MICROS;
+ } else if (Name.TIMESTAMP == type.name()) {
+ Types.TimestampType timestampType = (Types.TimestampType) type;
+ if (timestampType.hasTimeZone()) {
+ return TimestampWithTimeZoneType.TIMESTAMP_TZ_MICROS;
+ } else {
+ return TimestampType.TIMESTAMP_MICROS;
+ }
}
+
return super.getTrinoType(type);
}
}
diff --git
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/util/GeneralDataTypeTransformer.java
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/util/GeneralDataTypeTransformer.java
index 02fae0f7e..7da0f07a6 100644
---
a/trino-connector/src/main/java/org/apache/gravitino/trino/connector/util/GeneralDataTypeTransformer.java
+++
b/trino-connector/src/main/java/org/apache/gravitino/trino/connector/util/GeneralDataTypeTransformer.java
@@ -36,6 +36,7 @@ import io.trino.spi.type.RowType.Field;
import io.trino.spi.type.SmallintType;
import io.trino.spi.type.TimeType;
import io.trino.spi.type.TimestampType;
+import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.TinyintType;
import io.trino.spi.type.Type;
import io.trino.spi.type.TypeOperators;
@@ -103,7 +104,11 @@ public class GeneralDataTypeTransformer {
case TIME:
return TimeType.TIME_MILLIS;
case TIMESTAMP:
- return TimestampType.TIMESTAMP_MILLIS;
+ if (((Types.TimestampType) type).hasTimeZone()) {
+ return TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS;
+ } else {
+ return TimestampType.TIMESTAMP_MILLIS;
+ }
case LIST:
return new ArrayType(getTrinoType(((Types.ListType)
type).elementType()));
case MAP:
@@ -173,10 +178,12 @@ public class GeneralDataTypeTransformer {
return Types.BinaryType.get();
} else if (typeClass == io.trino.spi.type.DateType.class) {
return Types.DateType.get();
- } else if (typeClass == io.trino.spi.type.TimeType.class) {
+ } else if (io.trino.spi.type.TimeType.class.isAssignableFrom(typeClass)) {
return Types.TimeType.get();
} else if
(io.trino.spi.type.TimestampType.class.isAssignableFrom(typeClass)) {
return Types.TimestampType.withoutTimeZone();
+ } else if
(io.trino.spi.type.TimestampWithTimeZoneType.class.isAssignableFrom(typeClass))
{
+ return Types.TimestampType.withTimeZone();
} else if (typeClass == io.trino.spi.type.ArrayType.class) {
// Ignore nullability for the type, we could only get nullability from
column metadata
return Types.ListType.of(
diff --git
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java
index 87c03f39c..0bfbef3cf 100644
---
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java
+++
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/iceberg/TestIcebergDataTypeTransformer.java
@@ -20,6 +20,9 @@
package org.apache.gravitino.trino.connector.catalog.iceberg;
import io.trino.spi.TrinoException;
+import io.trino.spi.type.TimeType;
+import io.trino.spi.type.TimestampType;
+import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.VarcharType;
import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.trino.connector.util.GeneralDataTypeTransformer;
@@ -51,5 +54,16 @@ public class TestIcebergDataTypeTransformer {
Assertions.assertEquals(
generalDataTypeTransformer.getGravitinoType(varcharTypeWithoutLength),
Types.StringType.get());
+
+ Assertions.assertEquals(
+ generalDataTypeTransformer.getTrinoType(Types.TimeType.get()),
TimeType.TIME_MICROS);
+
+ Assertions.assertEquals(
+
generalDataTypeTransformer.getTrinoType(Types.TimestampType.withoutTimeZone()),
+ TimestampType.TIMESTAMP_MICROS);
+
+ Assertions.assertEquals(
+
generalDataTypeTransformer.getTrinoType(Types.TimestampType.withTimeZone()),
+ TimestampWithTimeZoneType.TIMESTAMP_TZ_MICROS);
}
}
diff --git
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/util/TestDataTypeTransformer.java
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/util/TestDataTypeTransformer.java
index d29c0697a..b5548170a 100644
---
a/trino-connector/src/test/java/org/apache/gravitino/trino/connector/util/TestDataTypeTransformer.java
+++
b/trino-connector/src/test/java/org/apache/gravitino/trino/connector/util/TestDataTypeTransformer.java
@@ -38,6 +38,7 @@ import io.trino.spi.type.RowType.Field;
import io.trino.spi.type.SmallintType;
import io.trino.spi.type.TimeType;
import io.trino.spi.type.TimestampType;
+import io.trino.spi.type.TimestampWithTimeZoneType;
import io.trino.spi.type.TinyintType;
import io.trino.spi.type.TypeOperators;
import io.trino.spi.type.UuidType;
@@ -91,6 +92,9 @@ public class TestDataTypeTransformer {
Assertions.assertEquals(
dataTypeTransformer.getGravitinoType(TimestampType.TIMESTAMP_MILLIS),
Types.TimestampType.withoutTimeZone());
+ Assertions.assertEquals(
+
dataTypeTransformer.getGravitinoType(TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS),
+ Types.TimestampType.withTimeZone());
Assertions.assertEquals(
dataTypeTransformer.getGravitinoType(new
ArrayType(IntegerType.INTEGER)),
@@ -168,6 +172,9 @@ public class TestDataTypeTransformer {
Assertions.assertEquals(
dataTypeTransformer.getTrinoType(Types.TimestampType.withoutTimeZone()),
TimestampType.TIMESTAMP_MILLIS);
+ Assertions.assertEquals(
+ dataTypeTransformer.getTrinoType(Types.TimestampType.withTimeZone()),
+ TimestampWithTimeZoneType.TIMESTAMP_TZ_MILLIS);
Assertions.assertEquals(
dataTypeTransformer.getTrinoType(Types.ListType.nullable(Types.IntegerType.get())),
