This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new d87ba33e3 [hive] HiveCatalog supports TIME type by converting time to
string (#1813)
d87ba33e3 is described below
commit d87ba33e3a875f48282987f0c796087afea5cf20
Author: yuzelin <[email protected]>
AuthorDate: Tue Aug 15 21:27:27 2023 +0800
[hive] HiveCatalog supports TIME type by converting time to string (#1813)
---
.../java/org/apache/paimon/hive/HiveTypeUtils.java | 6 +++
.../PaimonObjectInspectorFactory.java | 7 +++
.../objectinspector/PaimonTimeObjectInspector.java | 53 +++++++++++++++++++
.../apache/paimon/hive/HiveCatalogITCaseBase.java | 26 ----------
.../paimon/hive/PaimonStorageHandlerITCase.java | 52 +++++++++++++++++++
.../paimon/hive/RandomGenericRowDataGenerator.java | 5 ++
.../PaimonInternalRowObjectInspectorTest.java | 2 +
.../PaimonTimeObjectInspectorTest.java | 60 ++++++++++++++++++++++
8 files changed, 185 insertions(+), 26 deletions(-)
diff --git
a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java
index 986d6416a..79e79cbd5 100644
---
a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java
+++
b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java
@@ -37,6 +37,7 @@ import org.apache.paimon.types.MapType;
import org.apache.paimon.types.MultisetType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.SmallIntType;
+import org.apache.paimon.types.TimeType;
import org.apache.paimon.types.TimestampType;
import org.apache.paimon.types.TinyIntType;
import org.apache.paimon.types.VarBinaryType;
@@ -172,6 +173,11 @@ public class HiveTypeUtils {
return TypeInfoFactory.dateTypeInfo;
}
+ @Override
+ public TypeInfo visit(TimeType timeType) {
+ return TypeInfoFactory.stringTypeInfo;
+ }
+
@Override
public TypeInfo visit(TimestampType timestampType) {
return TypeInfoFactory.timestampTypeInfo;
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonObjectInspectorFactory.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonObjectInspectorFactory.java
index b8f798f41..fd556b54e 100644
---
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonObjectInspectorFactory.java
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonObjectInspectorFactory.java
@@ -26,6 +26,7 @@ import org.apache.paimon.types.DataType;
import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
+import org.apache.paimon.types.TimeType;
import org.apache.paimon.types.VarCharType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -35,6 +36,8 @@ import
org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import java.util.List;
import java.util.stream.Collectors;
+import static org.apache.paimon.utils.Preconditions.checkArgument;
+
/** Factory to create {@link ObjectInspector}s according to the given {@link
DataType}. */
public class PaimonObjectInspectorFactory {
@@ -67,6 +70,10 @@ public class PaimonObjectInspectorFactory {
}
case DATE:
return new PaimonDateObjectInspector();
+ case TIME_WITHOUT_TIME_ZONE:
+ TimeType timeType = (TimeType) logicalType;
+ checkArgument(timeType.getPrecision() <= 3, "TIME type
precision must be <= 3.");
+ return new PaimonTimeObjectInspector();
case TIMESTAMP_WITHOUT_TIME_ZONE:
return new PaimonTimestampObjectInspector();
case ARRAY:
diff --git
a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspector.java
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspector.java
new file mode 100644
index 000000000..e81a18afa
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspector.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.objectinspector;
+
+import org.apache.paimon.utils.DateTimeUtils;
+
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
+import
org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+
+/** {@link AbstractPrimitiveJavaObjectInspector} for TIME type. */
+public class PaimonTimeObjectInspector extends
AbstractPrimitiveJavaObjectInspector
+ implements StringObjectInspector, WriteableObjectInspector {
+
+ public PaimonTimeObjectInspector() {
+ super(TypeInfoFactory.stringTypeInfo);
+ }
+
+ @Override
+ public String getPrimitiveJavaObject(Object o) {
+ if (o == null) {
+ return null;
+ }
+ return DateTimeUtils.toLocalTime((int) o).toString();
+ }
+
+ @Override
+ public Text getPrimitiveWritableObject(Object o) {
+ return o == null ? null : new Text(getPrimitiveJavaObject(o));
+ }
+
+ @Override
+ public Integer convert(Object value) {
+ return value == null ? null : DateTimeUtils.parseTime((String) value);
+ }
+}
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
index c2ceba3bc..4ebf21136 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
@@ -27,7 +27,6 @@ import org.apache.paimon.fs.local.LocalFileIO;
import org.apache.paimon.hive.annotation.Minio;
import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.s3.MinioTestContainer;
-import org.apache.paimon.table.FileStoreTable;
import com.klarna.hiverunner.HiveShell;
import com.klarna.hiverunner.annotations.HiveSQL;
@@ -64,7 +63,6 @@ import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
@@ -750,30 +748,6 @@ public abstract class HiveCatalogITCaseBase {
assertThat(tableOptions).doesNotContainKey("lock.enabled");
}
- @Test
- public void testClearSchemaAfterUnSupportType()
- throws InterruptedException, ExecutionException,
Catalog.TableNotExistException {
- assertThatThrownBy(
- () ->
- tEnv.executeSql(
- "CREATE TABLE t001(id INT
PRIMARY KEY NOT ENFORCED , d TIME)")
- .await())
- .hasRootCauseInstanceOf(UnsupportedOperationException.class)
- .hasRootCauseMessage("Unsupported type: TIME(0)");
- Identifier identifier = new Identifier("test_db", "t001");
- Catalog catalog =
- ((FlinkCatalog)
tEnv.getCatalog(tEnv.getCurrentCatalog()).get()).catalog();
- assertThat(catalog.tableExists(identifier)).isFalse();
-
- tEnv.executeSql("CREATE TABLE t002(id INT PRIMARY KEY NOT ENFORCED ,
b STRING)").await();
- assertThatThrownBy(() -> tEnv.executeSql("ALTER TABLE t002 MODIFY b
TIME").await())
- .hasRootCauseInstanceOf(UnsupportedOperationException.class)
- .hasRootCauseMessage("Unsupported type: TIME(0)");
- identifier = new Identifier("test_db", "t002");
- FileStoreTable table = (FileStoreTable) catalog.getTable(identifier);
- assertThat(table.schema().fields().toString()).isEqualTo("[`id` INT
NOT NULL, `b` STRING]");
- }
-
@Test
public void testAddPartitionsToMetastore() throws Exception {
prepareTestAddPartitionsToMetastore();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
index 28ef6833d..17bb98bc8 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonStorageHandlerITCase.java
@@ -976,6 +976,58 @@ public class PaimonStorageHandlerITCase {
.containsExactly("NULL\t2022-06-18 08:30:00.1");
}
+ @Test
+ public void testTime() throws Exception {
+ Table table =
+ FileStoreTestUtils.createFileStoreTable(
+ getBasicConf(),
+ RowType.of(
+ new DataType[] {
+ DataTypes.INT().notNull(),
DataTypes.TIME(), DataTypes.TIME(2)
+ },
+ new String[] {"pk", "time0", "time2"}),
+ Collections.emptyList(),
+ Collections.singletonList("pk"));
+
+ StreamWriteBuilder streamWriteBuilder = table.newStreamWriteBuilder();
+ StreamTableWrite write = streamWriteBuilder.newWrite();
+ StreamTableCommit commit = streamWriteBuilder.newCommit();
+
+ write.write(GenericRow.of(1, 0, 0)); // 00:00
+ commit.commit(0, write.prepareCommit(true, 0));
+
+ write.write(GenericRow.of(2, 86_399_999, 86_399_999)); //
23:59:59.999999
+ commit.commit(1, write.prepareCommit(true, 1));
+
+ write.write(GenericRow.of(3, 45_001_000, 45_001_000)); // 12:30:01
+ commit.commit(2, write.prepareCommit(true, 2));
+
+ write.write(GenericRow.of(4, null, null));
+ commit.commit(4, write.prepareCommit(true, 3));
+
+ write.close();
+
+ createExternalTable();
+
+ // the TIME column will be converted to STRING column
+ assertThat(hiveShell.executeQuery("SHOW CREATE TABLE " +
externalTable))
+ .contains(
+ " `time0` string COMMENT 'from deserializer', ",
+ " `time2` string COMMENT 'from deserializer')");
+
+ assertThat(hiveShell.executeQuery("SELECT * FROM " + externalTable))
+ .containsExactlyInAnyOrder(
+ "1\t00:00\t00:00",
+ "2\t23:59:59.999\t23:59:59.999",
+ "3\t12:30:01\t12:30:01",
+ "4\tNULL\tNULL");
+
+ assertThat(
+ hiveShell.executeQuery(
+ "SELECT * FROM " + externalTable + " WHERE
time0 = '12:30:01'"))
+ .containsExactlyInAnyOrder("3\t12:30:01\t12:30:01");
+ }
+
@Test
public void testMapKey() throws Exception {
Options conf = getBasicConf();
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/RandomGenericRowDataGenerator.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/RandomGenericRowDataGenerator.java
index 74d4c92c0..9fb846a6b 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/RandomGenericRowDataGenerator.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/RandomGenericRowDataGenerator.java
@@ -57,6 +57,7 @@ public class RandomGenericRowDataGenerator {
DataTypes.STRING(),
DataTypes.VARBINARY(Integer.MAX_VALUE),
DataTypes.DATE(),
+ DataTypes.TIME(),
DataTypes.TIMESTAMP(3),
DataTypes.ARRAY(DataTypes.BIGINT()),
DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()));
@@ -77,6 +78,7 @@ public class RandomGenericRowDataGenerator {
"string",
"binary",
"date",
+ "string",
"timestamp",
"array<bigint>",
"map<string,int>");
@@ -97,6 +99,7 @@ public class RandomGenericRowDataGenerator {
"f_string",
"f_binary",
"f_date",
+ "f_time",
"f_timestamp",
"f_list_long",
"f_map_string_int");
@@ -117,6 +120,7 @@ public class RandomGenericRowDataGenerator {
"comment_string",
"comment_binary",
"comment_date",
+ "comment_time",
"comment_timestamp",
"comment_list_long",
"comment_map_string_int");
@@ -166,6 +170,7 @@ public class RandomGenericRowDataGenerator {
BinaryString.fromString(randomString(100)),
randomBytes,
random.nextInt(10000),
+ random.nextInt(86400),
Timestamp.fromEpochMillis(random.nextLong(Integer.MAX_VALUE)),
new GenericArray(randomLongArray),
new GenericMap(randomMap));
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonInternalRowObjectInspectorTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonInternalRowObjectInspectorTest.java
index 774e71df3..9930a62ce 100644
---
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonInternalRowObjectInspectorTest.java
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonInternalRowObjectInspectorTest.java
@@ -59,6 +59,7 @@ public class PaimonInternalRowObjectInspectorTest {
ObjectInspector.Category.PRIMITIVE,
ObjectInspector.Category.PRIMITIVE,
ObjectInspector.Category.PRIMITIVE,
+ ObjectInspector.Category.PRIMITIVE,
ObjectInspector.Category.LIST,
ObjectInspector.Category.MAP);
@@ -97,6 +98,7 @@ public class PaimonInternalRowObjectInspectorTest {
"f_string:string",
"f_binary:binary",
"f_date:date",
+ "f_time:string",
"f_timestamp:timestamp",
"f_list_long:array<bigint>",
"f_map_string_int:map<string,int>"))
diff --git
a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspectorTest.java
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspectorTest.java
new file mode 100644
index 000000000..2904434b9
--- /dev/null
+++
b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/objectinspector/PaimonTimeObjectInspectorTest.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.hive.objectinspector;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link PaimonTimeObjectInspector}. */
+public class PaimonTimeObjectInspectorTest {
+
+ @Test
+ public void testCategoryAndClass() {
+ PaimonTimeObjectInspector oi = new PaimonTimeObjectInspector();
+
+
assertThat(oi.getCategory()).isEqualTo(ObjectInspector.Category.PRIMITIVE);
+ assertThat(oi.getPrimitiveCategory())
+ .isEqualTo(PrimitiveObjectInspector.PrimitiveCategory.STRING);
+
+ assertThat(oi.getJavaPrimitiveClass()).isEqualTo(String.class);
+ assertThat(oi.getPrimitiveWritableClass()).isEqualTo(Text.class);
+ }
+
+ @Test
+ public void testGetPrimitiveJavaObject() {
+ PaimonTimeObjectInspector oi = new PaimonTimeObjectInspector();
+
+ int input = 1;
+ assertThat(oi.getPrimitiveJavaObject(input)).isEqualTo("00:00:00.001");
+ assertThat(oi.getPrimitiveJavaObject(null)).isNull();
+ }
+
+ @Test
+ public void testGetPrimitiveWritableObject() {
+ PaimonTimeObjectInspector oi = new PaimonTimeObjectInspector();
+
+ int input = 86_399_000;
+
assertThat(oi.getPrimitiveWritableObject(input).toString()).isEqualTo("23:59:59");
+ assertThat(oi.getPrimitiveWritableObject(null)).isNull();
+ }
+}
