This is an automated email from the ASF dual-hosted git repository. lzljs3620320 pushed a commit to branch release-0.4 in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
commit ab8ee352e4ecaf4eebe5293125eafa02fd9667cf Author: s7monk <[email protected]> AuthorDate: Tue Apr 25 14:45:24 2023 +0800 [hive] Support row type for hive catalog. (#1016) --- docs/content/engines/hive.md | 2 +- .../java/org/apache/paimon/hive/HiveTypeUtils.java | 19 ++++ .../org/apache/paimon/hive/HiveTypeUtilsTest.java | 106 +++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/docs/content/engines/hive.md b/docs/content/engines/hive.md index 446251fe9..9691e3386 100644 --- a/docs/content/engines/hive.md +++ b/docs/content/engines/hive.md @@ -193,7 +193,7 @@ All Hive's data types are available in package `org.apache.hadoop.hive.serde2.ty <thead> <tr> <th class="text-left" style="width: 10%">Hive Data Type</th> - <th class="text-left" style="width: 10%">Flink Data Type</th> + <th class="text-left" style="width: 10%">Paimon Data Type</th> <th class="text-left" style="width: 5%">Atomic Type</th> </tr> </thead> diff --git a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java index b33168e25..e70b645f3 100644 --- a/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java +++ b/paimon-hive/paimon-hive-common/src/main/java/org/apache/paimon/hive/HiveTypeUtils.java @@ -20,14 +20,19 @@ package org.apache.paimon.hive; import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.CharType; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DecimalType; import org.apache.paimon.types.MapType; +import org.apache.paimon.types.RowType; import org.apache.paimon.types.VarCharType; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory; +import java.util.List; +import java.util.stream.Collectors; + /** Utils for converting types related classes between Paimon and Hive. */ public class HiveTypeUtils { @@ -77,6 +82,20 @@ public class HiveTypeUtils { return TypeInfoFactory.getMapTypeInfo( logicalTypeToTypeInfo(mapType.getKeyType()), logicalTypeToTypeInfo(mapType.getValueType())); + + case ROW: + RowType rowType = (RowType) logicalType; + List<String> fieldNames = + rowType.getFields().stream() + .map(DataField::name) + .collect(Collectors.toList()); + List<TypeInfo> typeInfos = + rowType.getFields().stream() + .map(DataField::type) + .map(HiveTypeUtils::logicalTypeToTypeInfo) + .collect(Collectors.toList()); + return TypeInfoFactory.getStructTypeInfo(fieldNames, typeInfos); + default: throw new UnsupportedOperationException( "Unsupported logical type " + logicalType.asSQLString()); diff --git a/paimon-hive/paimon-hive-common/src/test/java/org/apache/paimon/hive/HiveTypeUtilsTest.java b/paimon-hive/paimon-hive-common/src/test/java/org/apache/paimon/hive/HiveTypeUtilsTest.java new file mode 100644 index 000000000..757b9d17a --- /dev/null +++ b/paimon-hive/paimon-hive-common/src/test/java/org/apache/paimon/hive/HiveTypeUtilsTest.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.hive; + +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.VarCharType; + +import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +/** Test for {@link HiveTypeUtils}. */ +public class HiveTypeUtilsTest { + + @Test + public void testLogicalTypeToTypeInfo() { + TypeInfo boolTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.BOOLEAN()); + assertThat(boolTypeInfo.getTypeName()).isEqualTo("boolean"); + + TypeInfo tinyintTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.TINYINT()); + assertThat(tinyintTypeInfo.getTypeName()).isEqualTo("tinyint"); + + TypeInfo smallintTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.SMALLINT()); + assertThat(smallintTypeInfo.getTypeName()).isEqualTo("smallint"); + + TypeInfo intTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.INT()); + assertThat(intTypeInfo.getTypeName()).isEqualTo("int"); + + TypeInfo bigintTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.BIGINT()); + assertThat(bigintTypeInfo.getTypeName()).isEqualTo("bigint"); + + TypeInfo floatTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.FLOAT()); + assertThat(floatTypeInfo.getTypeName()).isEqualTo("float"); + + TypeInfo doubleTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.DOUBLE()); + assertThat(doubleTypeInfo.getTypeName()).isEqualTo("double"); + + TypeInfo decimalTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.DECIMAL(38, 0)); + assertThat(decimalTypeInfo.getTypeName()).isEqualTo("decimal(38,0)"); + + TypeInfo decimal1TypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.DECIMAL(2, 2)); + assertThat(decimal1TypeInfo.getTypeName()).isEqualTo("decimal(2,2)"); + + TypeInfo charTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.CHAR(1)); + assertThat(charTypeInfo.getTypeName()).isEqualTo("char(1)"); + + TypeInfo varcharTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.VARCHAR(10)); + assertThat(varcharTypeInfo.getTypeName()).isEqualTo("varchar(10)"); + + TypeInfo binaryTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.BINARY(10)); + assertThat(binaryTypeInfo.getTypeName()).isEqualTo("binary"); + + TypeInfo varbinaryTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.VARBINARY(10)); + assertThat(varbinaryTypeInfo.getTypeName()).isEqualTo("binary"); + + TypeInfo dateTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.DATE()); + assertThat(dateTypeInfo.getTypeName()).isEqualTo("date"); + + TypeInfo timestampTypeInfo = HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.TIMESTAMP()); + assertThat(timestampTypeInfo.getTypeName()).isEqualTo("timestamp"); + + TypeInfo arrayTypeInfo = + HiveTypeUtils.logicalTypeToTypeInfo(DataTypes.ARRAY(DataTypes.INT())); + assertThat(arrayTypeInfo.getTypeName()).isEqualTo("array<int>"); + + TypeInfo mapTypeInfo = + HiveTypeUtils.logicalTypeToTypeInfo( + DataTypes.MAP(DataTypes.BIGINT(), DataTypes.STRING())); + assertThat(mapTypeInfo.getTypeName()).isEqualTo("map<bigint,string>"); + + TypeInfo rowTypeInfo = + HiveTypeUtils.logicalTypeToTypeInfo( + DataTypes.ROW( + new DataField(0, "id", new IntType()), + new DataField(1, "name", new VarCharType(Integer.MAX_VALUE)))); + assertThat(rowTypeInfo.getTypeName()).isEqualTo("struct<id:int,name:string>"); + + assertThatExceptionOfType(UnsupportedOperationException.class) + .isThrownBy( + () -> + HiveTypeUtils.logicalTypeToTypeInfo( + DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE()) + .getTypeName()) + .withMessage("Unsupported logical type TIMESTAMP(6) WITH LOCAL TIME ZONE"); + } +}
