This is an automated email from the ASF dual-hosted git repository.
difin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e14df633406 HIVE-29699: Iceberg: Add support for the Unknown data type
(#6588)
e14df633406 is described below
commit e14df63340649c48beadd65498120bf948c5cf2a
Author: Dmitriy Fingerman <[email protected]>
AuthorDate: Fri Jul 10 18:41:43 2026 -0400
HIVE-29699: Iceberg: Add support for the Unknown data type (#6588)
* HIVE-29699: Iceberg: Add support for the Unknown data type
---------
Co-authored-by: Dmitriy Fingerman <[email protected]>
Co-authored-by: Dmitriy Fingerman <[email protected]>
---
.../apache/iceberg/hive/HiveSchemaConverter.java | 2 +
.../org/apache/iceberg/hive/HiveSchemaUtil.java | 2 +
.../apache/iceberg/hive/TestHiveSchemaUtil.java | 30 +++
.../org/apache/iceberg/mr/hive/Deserializer.java | 10 +
.../objectinspector/IcebergObjectInspector.java | 3 +
.../iceberg/mr/hive/vector/HiveValueConverter.java | 2 +
.../mr/hive/TestHiveIcebergUnknownType.java | 89 +++++++++
.../TestIcebergObjectInspector.java | 12 +-
.../src/test/queries/positive/unknown_type.q | 54 ++++++
.../src/test/results/positive/unknown_type.q.out | 214 +++++++++++++++++++++
jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java | 4 +
.../org/apache/hadoop/hive/ql/parse/HiveParser.g | 9 +-
.../hive/ql/exec/vector/VectorAssignRow.java | 6 +
.../hive/ql/exec/vector/VectorDeserializeRow.java | 3 +
.../hive/ql/exec/vector/VectorExtractRow.java | 2 +
.../hive/ql/exec/vector/VectorSerializeRow.java | 11 ++
.../hive/ql/exec/vector/VectorizationContext.java | 2 +
.../hive/ql/exec/vector/VectorizedBatchUtil.java | 2 +
.../expressions/VectorExpressionWriterFactory.java | 25 +++
.../ql/io/parquet/convert/HiveSchemaConverter.java | 4 +-
.../org/apache/hadoop/hive/ql/metadata/Table.java | 2 +
.../calcite/translator/TypeConverter.java | 6 +
.../hadoop/hive/ql/parse/BaseSemanticAnalyzer.java | 1 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 7 +-
.../hadoop/hive/ql/stats/TestStatsUtils.java | 1 +
.../unknown_type_non_iceberg_table.q | 1 +
.../unknown_type_non_iceberg_table.q.out | 5 +
serde/if/serde.thrift | 1 +
.../apache/hadoop/hive/serde/serdeConstants.java | 1 +
.../hadoop/hive/serde2/lazy/LazyFactory.java | 5 +
.../hive/serde2/lazybinary/LazyBinaryUtils.java | 6 +
.../serde2/objectinspector/ObjectInspector.java | 2 +-
.../objectinspector/ObjectInspectorFactory.java | 4 +
.../objectinspector/UnknownObjectInspector.java | 46 +++++
.../org/apache/hadoop/hive/serde2/thrift/Type.java | 8 +-
.../hive/serde2/typeinfo/TypeInfoFactory.java | 4 +
.../hadoop/hive/serde2/typeinfo/TypeInfoUtils.java | 44 +++++
.../hive/serde2/typeinfo/UnknownTypeInfo.java | 69 +++++++
.../hive/serde2/typeinfo/TestUnknownTypeInfo.java | 79 ++++++++
service-rpc/if/TCLIService.thrift | 7 +-
.../service/rpc/thrift/TCLIServiceConstants.java | 2 +
.../apache/hive/service/rpc/thrift/TTypeId.java | 5 +-
service-rpc/src/gen/thrift/gen-php/TTypeId.php | 3 +
.../org/apache/hive/service/cli/ColumnValue.java | 2 +
.../apache/hadoop/hive/metastore/ColumnType.java | 2 +
45 files changed, 789 insertions(+), 10 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index 07631516cd4..e7a0ad91571 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -186,6 +186,8 @@ Type convertType(TypeInfo typeInfo, String defaultValue,
boolean shouldAddInitia
return Types.ListType.ofOptional(listId, listType);
case VARIANT:
return Types.VariantType.get();
+ case UNKNOWN:
+ return Types.UnknownType.get();
default:
throw new IllegalArgumentException("Unknown type " +
typeInfo.getCategory());
}
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
index 3f18deab4ea..306fb343e22 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
@@ -411,6 +411,8 @@ public static String convertToTypeString(Type type) {
return String.format("map<%s,%s>", convert(mapType.keyType()),
convert(mapType.valueType()));
case VARIANT:
return "variant";
+ case UNKNOWN:
+ return "unknown";
default:
throw new UnsupportedOperationException(type + " is not supported");
}
diff --git
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
index 6234a6f8db5..838344ac62d 100644
---
a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
+++
b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestHiveSchemaUtil.java
@@ -26,6 +26,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -171,6 +172,35 @@ public void testConversionWithoutLastComment() {
assertThat(schema.asStruct()).isEqualTo(expected.asStruct());
}
+ @Test
+ public void testUnknownTypeConvertToIcebergSchema() {
+ List<FieldSchema> hiveSchema = ImmutableList.of(
+ new FieldSchema("id", serdeConstants.INT_TYPE_NAME, null),
+ new FieldSchema("placeholder", serdeConstants.UNKNOWN_TYPE_NAME,
"placeholder comment"));
+ Schema expected = new Schema(
+ optional(1, "id", Types.IntegerType.get()),
+ optional(2, "placeholder", Types.UnknownType.get(), "placeholder
comment"));
+
+
assertThat(HiveSchemaUtil.convert(hiveSchema).asStruct()).isEqualTo(expected.asStruct());
+ }
+
+ @Test
+ public void testUnknownTypeConvertToHiveSchema() {
+ Schema icebergSchema = new Schema(
+ optional(1, "id", Types.IntegerType.get()),
+ optional(2, "placeholder", Types.UnknownType.get(), "placeholder
comment"));
+ List<FieldSchema> expected = ImmutableList.of(
+ new FieldSchema("id", serdeConstants.INT_TYPE_NAME, null),
+ new FieldSchema("placeholder", serdeConstants.UNKNOWN_TYPE_NAME,
"placeholder comment"));
+
+ assertThat(HiveSchemaUtil.convert(icebergSchema)).isEqualTo(expected);
+ }
+
+ @Test
+ public void testUnknownTypeAndTypeInfoConvert() {
+ checkConvert(TypeInfoFactory.getUnknownTypeInfo(),
Types.UnknownType.get());
+ }
+
protected List<FieldSchema> getSupportedFieldSchemas() {
List<FieldSchema> fields = Lists.newArrayList();
fields.add(new FieldSchema("c_float", serdeConstants.FLOAT_TYPE_NAME,
"float comment"));
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java
index dbafe176f6b..95bc7fd650f 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/Deserializer.java
@@ -37,6 +37,7 @@
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.schema.SchemaWithPartnerVisitor;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Type.PrimitiveType;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.types.Types.ListType;
@@ -110,6 +111,15 @@ public FieldDeserializer field(NestedField field,
ObjectInspectorPair pair, Fiel
@Override
public FieldDeserializer primitive(PrimitiveType type, ObjectInspectorPair
pair) {
+ if (type.typeId() == Type.TypeID.UNKNOWN) {
+ return o -> {
+ if (o != null) {
+ throw new IllegalArgumentException("UNKNOWN type columns only
accept NULL values");
+ }
+ return null;
+ };
+ }
+
return o -> {
if (o == null) {
return null;
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
index 6f4b563fa5f..bf5158d62a5 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
@@ -23,6 +23,7 @@
import javax.annotation.Nullable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -106,6 +107,8 @@ public ObjectInspector primitive(Type.PrimitiveType
primitiveType) {
IcebergTimestampObjectInspectorHive3.get(9);
case TIME:
return IcebergTimeObjectInspector.get();
+ case UNKNOWN:
+ return UnknownObjectInspector.get();
default:
throw new IllegalArgumentException(primitiveType.typeId() + " type is
not supported");
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java
index 0804a7abe9f..e5e61c46606 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/vector/HiveValueConverter.java
@@ -104,6 +104,8 @@ public static Object convert(Type type, Object object) {
case DOUBLE:
case STRING:
return object;
+ case UNKNOWN:
+ return null;
default:
throw new UnsupportedOperationException("Not a supported type: " +
type);
}
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java
new file mode 100644
index 00000000000..fd2647964c6
--- /dev/null
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergUnknownType.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.util.Collection;
+import java.util.List;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.mr.hive.test.TestTables.TestTableType;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runners.Parameterized.Parameters;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+
+public class TestHiveIcebergUnknownType extends
HiveIcebergStorageHandlerWithEngineBase {
+
+ @Parameters(name = "fileFormat={0}, catalog={1}, isVectorized={2},
formatVersion={3}")
+ public static Collection<Object[]> parameters() {
+ return HiveIcebergStorageHandlerWithEngineBase.getParameters(p ->
+ p.fileFormat() == FileFormat.PARQUET &&
+ p.testTableType() == TestTableType.HIVE_CATALOG &&
+ p.formatVersion() == 3);
+ }
+
+ @Test
+ public void testUnknownTypeReadWrite() {
+ Schema schema = new Schema(
+ optional(1, "id", Types.IntegerType.get()),
+ optional(2, "placeholder", Types.UnknownType.get()));
+
+ TableIdentifier table = TableIdentifier.of("default",
"unknown_type_basic");
+
+ testTables.createTable(shell, table.name(),
+ schema, org.apache.iceberg.PartitionSpec.unpartitioned(), fileFormat,
ImmutableList.of(), formatVersion,
+ ImmutableMap.of());
+
+ shell.executeStatement(
+ String.format("INSERT INTO %s VALUES (1, NULL), (2, NULL)", table));
+
+ List<Object[]> rows = shell.executeStatement(
+ String.format("SELECT id, placeholder FROM %s ORDER BY id", table));
+
+ Assert.assertEquals(2, rows.size());
+ Assert.assertEquals(1, rows.get(0)[0]);
+ Assert.assertNull(rows.get(0)[1]);
+ Assert.assertEquals(2, rows.get(1)[0]);
+ Assert.assertNull(rows.get(1)[1]);
+ }
+
+ @Test
+ public void testUnknownTypeDdl() {
+ TableIdentifier table = TableIdentifier.of("default", "unknown_type_ddl");
+
+ shell.executeStatement(String.format(
+ "CREATE EXTERNAL TABLE %s (id INT, placeholder UNKNOWN) STORED BY
ICEBERG " +
+ "TBLPROPERTIES ('format-version'='3')", table));
+
+ shell.executeStatement(String.format("INSERT INTO %s VALUES (1, NULL)",
table));
+
+ List<Object[]> rows = shell.executeStatement(
+ String.format("SELECT id, placeholder FROM %s", table));
+
+ Assert.assertEquals(1, rows.size());
+ Assert.assertEquals(1, rows.get(0)[0]);
+ Assert.assertNull(rows.get(0)[1]);
+ }
+}
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
index a4f2dcc10f5..7e9a3eb6f47 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -32,6 +33,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import static org.apache.iceberg.types.Types.NestedField.optional;
import static org.apache.iceberg.types.Types.NestedField.required;
@@ -60,7 +62,8 @@ public class TestIcebergObjectInspector {
Types.NestedField.required(20, "nested_field",
Types.StringType.get(), "nested field comment")),
"struct comment"
),
- required(21, "time_field", Types.TimeType.get(), "time comment")
+ required(21, "time_field", Types.TimeType.get(), "time comment"),
+ optional(22, "unknown_field", Types.UnknownType.get(), "unknown
comment")
);
@Test
@@ -205,6 +208,13 @@ public void testIcebergObjectInspector() {
Assertions.assertEquals("time_field", timeField.getFieldName());
Assertions.assertEquals("time comment", timeField.getFieldComment());
Assertions.assertEquals(IcebergTimeObjectInspector.get(),
timeField.getFieldObjectInspector());
+
+ // unknown
+ StructField unknownField = soi.getStructFieldRef("unknown_field");
+ Assertions.assertEquals(22, unknownField.getFieldID());
+ Assertions.assertEquals("unknown_field", unknownField.getFieldName());
+ Assertions.assertEquals("unknown comment", unknownField.getFieldComment());
+ Assertions.assertEquals(UnknownObjectInspector.get(),
unknownField.getFieldObjectInspector());
}
private static ObjectInspector getPrimitiveObjectInspector(Class<?> clazz) {
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q
b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q
new file mode 100644
index 00000000000..60c9edad983
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/queries/positive/unknown_type.q
@@ -0,0 +1,54 @@
+-- SORT_QUERY_RESULTS
+-- Mask neededVirtualColumns due to non-strict order
+--! qt:replace:/(\s+neededVirtualColumns:\s)(.*)/$1#Masked#/
+-- Mask random uuid
+--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask a random snapshot id
+--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/
+-- Mask added file size
+--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask total file size
+--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask current-snapshot-timestamp-ms
+--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/
+-- Mask removed file size
+--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/
+-- Mask iceberg version
+--!
qt:replace:/(\S\"iceberg-version\\\":\\\")(\w+\s\w+\s\d+\.\d+\.\d+\s\(\w+\s\w+\))(\\\")/$1#Masked#$3/
+
+-- Create test table with unknown placeholder column
+CREATE EXTERNAL TABLE unknown_test_basic (
+ id INT,
+ placeholder UNKNOWN
+) STORED BY ICEBERG tblproperties('format-version'='3');
+
+-- Unknown columns are not stored; only NULL values are accepted
+INSERT INTO unknown_test_basic VALUES
+(1, NULL),
+(2, NULL);
+
+SELECT id, placeholder FROM unknown_test_basic ORDER BY id;
+
+-- Add another unknown column to an existing table
+ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN);
+
+INSERT INTO unknown_test_basic VALUES
+(3, NULL, NULL);
+
+SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY id;
+
+-- CTLT preserves unknown column types from the source table
+CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED BY ICEBERG
tblproperties('format-version'='3');
+
+DESC FORMATTED unknown_test_ctlt;
+
+INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic;
+
+SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY id;
+
+-- CTAS preserves unknown column types from the query schema
+CREATE TABLE unknown_test_ctas STORED BY ICEBERG
tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic;
+
+DESC FORMATTED unknown_test_ctas;
+
+SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY id;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out
b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out
new file mode 100644
index 00000000000..faa92ff4836
--- /dev/null
+++ b/iceberg/iceberg-handler/src/test/results/positive/unknown_type.q.out
@@ -0,0 +1,214 @@
+PREHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic (
+ id INT,
+ placeholder UNKNOWN
+) STORED BY ICEBERG tblproperties('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unknown_test_basic
+POSTHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic (
+ id INT,
+ placeholder UNKNOWN
+) STORED BY ICEBERG tblproperties('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unknown_test_basic
+PREHOOK: query: INSERT INTO unknown_test_basic VALUES
+(1, NULL),
+(2, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@unknown_test_basic
+POSTHOOK: query: INSERT INTO unknown_test_basic VALUES
+(1, NULL),
+(2, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@unknown_test_basic
+PREHOOK: query: SELECT id, placeholder FROM unknown_test_basic ORDER BY id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unknown_test_basic
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, placeholder FROM unknown_test_basic ORDER BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unknown_test_basic
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL
+2 NULL
+PREHOOK: query: ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN)
+PREHOOK: type: ALTERTABLE_ADDCOLS
+PREHOOK: Input: default@unknown_test_basic
+PREHOOK: Output: default@unknown_test_basic
+POSTHOOK: query: ALTER TABLE unknown_test_basic ADD COLUMNS (extra UNKNOWN)
+POSTHOOK: type: ALTERTABLE_ADDCOLS
+POSTHOOK: Input: default@unknown_test_basic
+POSTHOOK: Output: default@unknown_test_basic
+PREHOOK: query: INSERT INTO unknown_test_basic VALUES
+(3, NULL, NULL)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@unknown_test_basic
+POSTHOOK: query: INSERT INTO unknown_test_basic VALUES
+(3, NULL, NULL)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@unknown_test_basic
+PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_basic ORDER BY
id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unknown_test_basic
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_basic ORDER
BY id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unknown_test_basic
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL
+2 NULL NULL
+3 NULL NULL
+PREHOOK: query: CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED
BY ICEBERG tblproperties('format-version'='3')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unknown_test_ctlt
+POSTHOOK: query: CREATE TABLE unknown_test_ctlt LIKE unknown_test_basic STORED
BY ICEBERG tblproperties('format-version'='3')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unknown_test_ctlt
+PREHOOK: query: DESC FORMATTED unknown_test_ctlt
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@unknown_test_ctlt
+POSTHOOK: query: DESC FORMATTED unknown_test_ctlt
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@unknown_test_ctlt
+# col_name data_type comment
+id int
+placeholder unknown
+extra unknown
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE
{\"BASIC_STATS\":\"true\",\"COLUMN_STATS\":{\"extra\":\"true\",\"id\":\"true\",\"placeholder\":\"true\"}}
+ EXTERNAL TRUE
+ TRANSLATED_TO_EXTERNAL TRUE
+ bucketing_version 2
+ created_with_ctlt true
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]}
+ format-version 3
+ metadata_location hdfs://### HDFS PATH ###
+ numFiles 0
+ numRows 0
+ parquet.compression zstd
+ rawDataSize 0
+ snapshot-count 0
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.merge.mode merge-on-read
+ write.metadata.delete-after-commit.enabled true
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unknown_test_basic
+PREHOOK: Output: default@unknown_test_ctlt
+POSTHOOK: query: INSERT INTO unknown_test_ctlt SELECT * FROM unknown_test_basic
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unknown_test_basic
+POSTHOOK: Output: default@unknown_test_ctlt
+PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY
id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unknown_test_ctlt
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctlt ORDER BY
id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unknown_test_ctlt
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL
+2 NULL NULL
+3 NULL NULL
+PREHOOK: query: CREATE TABLE unknown_test_ctas STORED BY ICEBERG
tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@unknown_test_basic
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unknown_test_ctas
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: CREATE TABLE unknown_test_ctas STORED BY ICEBERG
tblproperties('format-version'='3') AS SELECT * FROM unknown_test_basic
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@unknown_test_basic
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@unknown_test_ctas
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: Lineage: unknown_test_ctas.extra SIMPLE
[(unknown_test_basic)unknown_test_basic.FieldSchema(name:extra, type:unknown,
comment:null), ]
+POSTHOOK: Lineage: unknown_test_ctas.id SIMPLE
[(unknown_test_basic)unknown_test_basic.FieldSchema(name:id, type:int,
comment:null), ]
+POSTHOOK: Lineage: unknown_test_ctas.placeholder SIMPLE
[(unknown_test_basic)unknown_test_basic.FieldSchema(name:placeholder,
type:unknown, comment:null), ]
+PREHOOK: query: DESC FORMATTED unknown_test_ctas
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@unknown_test_ctas
+POSTHOOK: query: DESC FORMATTED unknown_test_ctas
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@unknown_test_ctas
+# col_name data_type comment
+id int
+placeholder unknown
+extra unknown
+
+# Detailed Table Information
+Database: default
+#### A masked pattern was here ####
+Retention: 0
+#### A masked pattern was here ####
+Table Type: EXTERNAL_TABLE
+Table Parameters:
+ COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"}
+ EXTERNAL TRUE
+ bucketing_version 2
+ current-schema
{\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"placeholder\",\"required\":false,\"type\":\"unknown\"},{\"id\":3,\"name\":\"extra\",\"required\":false,\"type\":\"unknown\"}]}
+ current-snapshot-id #Masked#
+ current-snapshot-summary
{\"added-data-files\":\"1\",\"added-records\":\"3\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"3\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"1\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"#Masked#\"}
+ current-snapshot-timestamp-ms #Masked#
+ format-version 3
+ metadata_location hdfs://### HDFS PATH ###
+ numFiles 1
+ numRows 3
+ parquet.compression zstd
+ previous_metadata_location hdfs://### HDFS PATH ###
+ serialization.format 1
+ snapshot-count 1
+ storage_handler
org.apache.iceberg.mr.hive.HiveIcebergStorageHandler
+ table_type ICEBERG
+ totalSize #Masked#
+#### A masked pattern was here ####
+ uuid #Masked#
+ write.delete.mode merge-on-read
+ write.merge.mode merge-on-read
+ write.metadata.delete-after-commit.enabled true
+ write.update.mode merge-on-read
+
+# Storage Information
+SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe
+InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat
+OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat
+Compressed: No
+Sort Columns: []
+PREHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY
id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@unknown_test_ctas
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT id, placeholder, extra FROM unknown_test_ctas ORDER BY
id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@unknown_test_ctas
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 NULL NULL
+2 NULL NULL
+3 NULL NULL
diff --git a/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java
b/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java
index d3b7098fa7f..2747ae751d4 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/JdbcColumn.java
@@ -168,6 +168,8 @@ static Type typeStringToHiveType(String type) throws
SQLException {
/* fall through */
case "null":
return Type.NULL_TYPE;
+ case serdeConstants.UNKNOWN_TYPE_NAME:
+ return Type.UNKNOWN_TYPE;
default:
throw new SQLException("Unrecognized column type: " + type);
}
@@ -224,6 +226,8 @@ static String getColumnTypeName(String type) throws
SQLException {
/* fall through */
case "null":
return serdeConstants.VOID_TYPE_NAME;
+ case serdeConstants.UNKNOWN_TYPE_NAME:
+ return serdeConstants.UNKNOWN_TYPE_NAME;
case "map":
return serdeConstants.MAP_TYPE_NAME;
case "array":
diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 0033aa3251c..fba3f297221 100644
--- a/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -2420,7 +2420,8 @@ type
| structType
| mapType
| unionType
- | variantType;
+ | variantType
+ | unknownType;
primitiveType
@init { pushMsg("primitive type specification", state); }
@@ -2486,6 +2487,12 @@ variantType
: KW_VARIANT -> TOK_VARIANT
;
+unknownType
+@init { pushMsg("unknown type", state); }
+@after { popMsg(state); }
+ : KW_UNKNOWN -> TOK_UNKNOWN
+ ;
+
setOperator
@init { pushMsg("set operator", state); }
@after { popMsg(state); }
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
index 5e3768660f6..9f73060cc0b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorAssignRow.java
@@ -626,6 +626,9 @@ private void assignRowColumn(
assignRowColumn(unionColumnVector.fields[tag], batchIndex,
objectTypeInfos.get(tag), union.getObject());
}
break;
+ case UNKNOWN:
+ VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
+ return;
default:
throw new RuntimeException("Category " +
targetTypeInfo.getCategory().name() + " not supported");
}
@@ -986,6 +989,9 @@ private void assignConvertRowColumn(ColumnVector
columnVector, int batchIndex,
unionObjectInspector.getField(tag));
}
break;
+ case UNKNOWN:
+ VectorizedBatchUtil.setNullColIsNullValue(columnVector, batchIndex);
+ return;
default:
throw new RuntimeException("Category " + targetCategory.name() + " not
supported");
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
index c1f5d8ca32c..6410ff80e93 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorDeserializeRow.java
@@ -355,6 +355,8 @@ private Field allocateComplexField(TypeInfo sourceTypeInfo)
{
new UnionComplexTypeHelper(fields);
return new Field(category, unionHelper, sourceTypeInfo, new
VectorUnionDeserializer());
}
+ case UNKNOWN:
+ return new Field(category, null, sourceTypeInfo, new
VectorVoidDeserializer());
default:
throw new RuntimeException("Category " + category + " not supported");
}
@@ -368,6 +370,7 @@ private Field allocateField(TypeInfo sourceTypeInfo) {
case MAP:
case STRUCT:
case UNION:
+ case UNKNOWN:
return allocateComplexField(sourceTypeInfo);
default:
throw new RuntimeException("Category " + sourceTypeInfo.getCategory() +
" not supported");
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
index b3be81646c5..9945f957ef5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorExtractRow.java
@@ -564,6 +564,8 @@ public Object extractRowColumn(
standardUnion.setObject(object);
return standardUnion;
}
+ case UNKNOWN:
+ return null;
default:
throw new RuntimeException("Category " + category.name() + " not
supported");
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
index fd4fffe292f..deca7290800 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSerializeRow.java
@@ -176,6 +176,10 @@ private Field createField(TypeInfo typeInfo) {
field.children = createFields(objectTypeInfos.toArray(new
TypeInfo[objectTypeInfos.size()]));
field.writer = new VectorSerializeUnionWriter();
break;
+ case UNKNOWN:
+ field.children = new Field[0];
+ field.writer = new VectorSerializeUnknownWriter();
+ break;
default:
throw new RuntimeException();
}
@@ -579,6 +583,13 @@ void serialize(Object colInfo, Field field, int
adjustedBatchIndex) throws IOExc
}
}
+ class VectorSerializeUnknownWriter extends VectorSerializeWriter {
+ @Override
+ void serialize(Object colInfo, Field field, int adjustedBatchIndex) throws
IOException {
+ // Unknown type values are always null and handled before the writer is
invoked.
+ }
+ }
+
public boolean getHasAnyNulls() {
return hasAnyNulls;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 894231e991e..c6d2625eb77 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -4581,6 +4581,8 @@ public static ColumnVector.Type
getColumnVectorTypeFromTypeInfo(TypeInfo typeInf
throw new HiveException("Unexpected primitive type category " +
primitiveCategory);
}
}
+ case UNKNOWN:
+ return ColumnVector.Type.VOID;
default:
throw new HiveException("Unexpected type category " +
typeInfo.getCategory());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index 4be9e1435f0..d4d4af31a94 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -208,6 +208,8 @@ public static ColumnVector createColumnVector(TypeInfo
typeInfo,
createColumnVector(mapTypeInfo.getMapKeyTypeInfo()),
createColumnVector(mapTypeInfo.getMapValueTypeInfo()));
}
+ case UNKNOWN:
+ return new VoidColumnVector(VectorizedRowBatch.DEFAULT_SIZE);
default:
throw new RuntimeException("Vectorization is not supported for datatype:"
+ typeInfo.getCategory());
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
index 163347febeb..4c70bac5426 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
@@ -738,6 +738,9 @@ public static VectorExpressionWriter
genVectorExpressionWritable(
return genVectorExpressionWritableUnion(
(SettableUnionObjectInspector) fieldObjInspector);
+ case UNKNOWN:
+ return genVectorExpressionWritableUnknown(fieldObjInspector);
+
default:
throw new IllegalArgumentException("Unknown type " +
fieldObjInspector.getCategory());
@@ -1719,6 +1722,28 @@ public Object setValue(Object union, ColumnVector
column, int row)
}.init(fieldObjInspector);
}
+ private static VectorExpressionWriter genVectorExpressionWritableUnknown(
+ ObjectInspector fieldObjInspector) throws HiveException {
+ return new VectorExpressionWriterBase() {
+
+ @Override
+ public Object writeValue(ColumnVector column, int row) throws
HiveException {
+ return null;
+ }
+
+ @Override
+ public Object setValue(Object row, ColumnVector column, int columnRow)
+ throws HiveException {
+ return null;
+ }
+
+ @Override
+ public Object initValue(Object ost) throws HiveException {
+ return null;
+ }
+ }.init(fieldObjInspector);
+ }
+
// For complex types like STRUCT, MAP, etc we do not support, we need a
writer that
// does nothing. We assume the Vectorizer class has not validated the query
to actually
// try and use the complex types. They do show up in inputObjInspector[0]
and need to be
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
index a59c15ee5d8..e1ca092a20a 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveSchemaConverter.java
@@ -129,7 +129,9 @@ private static Type convertType(final String name, final
TypeInfo typeInfo, Conf
return Types.primitive(PrimitiveTypeName.INT32, repetition)
.as(LogicalTypeAnnotation.dateType()).named(name);
} else if (typeInfo.equals(TypeInfoFactory.unknownTypeInfo)) {
- throw new UnsupportedOperationException("Unknown type not
implemented");
+ throw new UnsupportedOperationException("Legacy unknown primitive type
is not supported for Parquet");
+ } else if (typeInfo.equals(TypeInfoFactory.getUnknownTypeInfo())) {
+ throw new UnsupportedOperationException("Unknown type is not stored in
Parquet data files");
} else {
throw new IllegalArgumentException("Unknown type: " + typeInfo);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
index ad16a2f22ce..001eff15f55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
@@ -93,6 +93,7 @@
import com.google.common.base.Preconditions;
import static org.apache.commons.lang3.StringUtils.isBlank;
+import static org.apache.hadoop.hive.serde.serdeConstants.UNKNOWN_TYPE_NAME;
import static org.apache.hadoop.hive.serde.serdeConstants.VARIANT_TYPE_NAME;
/**
@@ -1479,6 +1480,7 @@ public List<VirtualColumn> getVirtualColumns() {
private static boolean isUnsupportedInNonIceberg(String columnType) {
return VARIANT_TYPE_NAME.equalsIgnoreCase(columnType) ||
+ UNKNOWN_TYPE_NAME.equalsIgnoreCase(columnType) ||
TypeInfoFactory.nanoTimestampTypeInfo.getQualifiedName().equalsIgnoreCase(columnType)
||
TypeInfoFactory.timestampNanoLocalTZTypeInfo.getQualifiedName().equalsIgnoreCase(columnType);
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
index 1f7ccd1773c..9fa5ebf286d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/TypeConverter.java
@@ -164,6 +164,12 @@ public static RelDataType convert(TypeInfo type, boolean
nullable, RelDataTypeFa
case UNION:
convertedType = convert((UnionTypeInfo) type, dtFactory);
break;
+ case VARIANT:
+ convertedType = dtFactory.createSqlType(SqlTypeName.OTHER);
+ break;
+ case UNKNOWN:
+ convertedType = dtFactory.createSqlType(SqlTypeName.NULL);
+ break;
}
// hive does not have concept of not nullable types
return dtFactory.createTypeWithNullability(convertedType, nullable);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
index 8f763b4857c..434e131b4d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
@@ -1048,6 +1048,7 @@ public static String getTypeStringFromAST(ASTNode
typeNode)
TOKEN_TO_TYPE.put(HiveParser.TOK_INTERVAL_DAY_TIME,
serdeConstants.INTERVAL_DAY_TIME_TYPE_NAME);
TOKEN_TO_TYPE.put(HiveParser.TOK_DECIMAL,
serdeConstants.DECIMAL_TYPE_NAME);
TOKEN_TO_TYPE.put(HiveParser.TOK_VARIANT,
serdeConstants.VARIANT_TYPE_NAME);
+ TOKEN_TO_TYPE.put(HiveParser.TOK_UNKNOWN,
serdeConstants.UNKNOWN_TYPE_NAME);
}
private static String getTypeName(ASTNode node) throws SemanticException {
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index ea981c98bb3..b754e677a47 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -9126,6 +9126,9 @@ private ExprNodeDesc handleConversion(StructField
tableField, ColumnInfo rowFiel
// JSON-format. However, we may add more operators.
// Thus, we still keep the conversion.
if (!tableFieldTypeInfo.equals(rowFieldTypeInfo)) {
+ if (TypeInfoUtils.isVoidCompatibleTarget(rowFieldTypeInfo,
tableFieldTypeInfo)) {
+ return column;
+ }
// need to do some conversions here
conversion.set(true);
if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) {
@@ -9348,8 +9351,8 @@ private List<ExprNodeDesc> genConvertCol(String dest, QB
qb, TableDesc tableDesc
rowFields.get(posn).getInternalName(),
rowFields.get(posn).getTabAlias(),
rowFields.get(posn).getIsVirtualCol());
- if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo)) {
- // need to do some conversions here
+ if (convert && !tableFieldTypeInfo.equals(rowFieldTypeInfo)
+ && !TypeInfoUtils.isVoidCompatibleTarget(rowFieldTypeInfo,
tableFieldTypeInfo)) {
if (tableFieldTypeInfo.getCategory() != Category.PRIMITIVE) {
// cannot convert to complex types
column = null;
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
index 2faefafdee3..bb09151e788 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/stats/TestStatsUtils.java
@@ -108,6 +108,7 @@ void testPrimitiveSizeEstimations() throws Exception {
exclusions.add(serdeConstants.STRUCT_TYPE_NAME);
exclusions.add(serdeConstants.UNION_TYPE_NAME);
exclusions.add(serdeConstants.VARIANT_TYPE_NAME);
+ exclusions.add(serdeConstants.UNKNOWN_TYPE_NAME);
Field[] serdeFields = serdeConstants.class.getFields();
for (Field field : serdeFields) {
if (!Modifier.isStatic(field.getModifiers())) {
diff --git
a/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q
b/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q
new file mode 100644
index 00000000000..85084d25713
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/unknown_type_non_iceberg_table.q
@@ -0,0 +1 @@
+CREATE EXTERNAL TABLE unknown_test_basic (id INT, placeholder UNKNOWN);
diff --git
a/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out
b/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out
new file mode 100644
index 00000000000..597c0f28acf
--- /dev/null
+++ b/ql/src/test/results/clientnegative/unknown_type_non_iceberg_table.q.out
@@ -0,0 +1,5 @@
+PREHOOK: query: CREATE EXTERNAL TABLE unknown_test_basic (id INT, placeholder
UNKNOWN)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@unknown_test_basic
+FAILED: Execution Error, return code 40000 from
org.apache.hadoop.hive.ql.ddl.DDLTask.
org.apache.hadoop.hive.ql.metadata.HiveException: Column name placeholder
cannot be of type 'unknown' as it is not supported in non-Iceberg tables.
diff --git a/serde/if/serde.thrift b/serde/if/serde.thrift
index f8d0f0a06ac..057f227b75e 100644
--- a/serde/if/serde.thrift
+++ b/serde/if/serde.thrift
@@ -71,6 +71,7 @@ const string MAP_TYPE_NAME = "map";
const string STRUCT_TYPE_NAME = "struct";
const string UNION_TYPE_NAME = "uniontype";
const string VARIANT_TYPE_NAME = "variant";
+const string UNKNOWN_TYPE_NAME = "unknown";
const string LIST_COLUMNS = "columns";
const string LIST_COLUMN_TYPES = "columns.types";
diff --git
a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
index 20e607a67bd..d43bc3d8d96 100644
---
a/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
+++
b/serde/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/serde/serdeConstants.java
@@ -94,6 +94,7 @@ public class serdeConstants {
public static final java.lang.String UNION_TYPE_NAME = "uniontype";
public static final java.lang.String VARIANT_TYPE_NAME = "variant";
+ public static final java.lang.String UNKNOWN_TYPE_NAME = "unknown";
public static final java.lang.String LIST_COLUMNS = "columns";
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
index 8b9c012d9b9..4a51e6c90dd 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
@@ -68,6 +68,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -370,6 +371,10 @@ public static ObjectInspector
createLazyObjectInspector(TypeInfo typeInfo,
return LazyObjectInspectorFactory.getLazyUnionObjectInspector(lazyOIs,
LazyUtils.getSeparator(lazyParams.getSeparators(), separatorIndex),
lazyParams);
+ case UNKNOWN:
+ // Unknown columns are always null and are not stored in data files.
+ return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(
+ TypeInfoFactory.voidTypeInfo, lazyParams);
}
throw new RuntimeException("Hive LazySerDe Internal error.");
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
index ed521eed68b..f801a42b112 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryUtils.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.io.WritableUtils;
@@ -513,6 +514,11 @@ public static ObjectInspector
getLazyBinaryObjectInspectorFromTypeInfo(
.getLazyBinaryUnionObjectInspector(fieldObjectInspectors);
break;
}
+ case UNKNOWN:
+ // Unknown columns are always null and are not stored in data files.
+ result =
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
+ TypeInfoFactory.voidTypeInfo);
+ break;
default: {
result = null;
}
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
index 8e4183b615f..c342098c3e4 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspector.java
@@ -47,7 +47,7 @@ public interface ObjectInspector extends Cloneable {
*
*/
enum Category {
- PRIMITIVE, LIST, MAP, STRUCT, UNION, VARIANT
+ PRIMITIVE, LIST, MAP, STRUCT, UNION, VARIANT, UNKNOWN
};
/**
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
index 8e251d19bb3..9f6247466b1 100644
---
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
@@ -367,6 +367,10 @@ public static ObjectInspector getVariantObjectInspector() {
return VariantObjectInspector.get();
}
+ public static ObjectInspector getUnknownObjectInspector() {
+ return UnknownObjectInspector.get();
+ }
+
public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
List<String> structFieldNames,
List<ObjectInspector> structFieldObjectInspectors) {
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java
new file mode 100644
index 00000000000..0facbd6c810
--- /dev/null
+++
b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/UnknownObjectInspector.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.objectinspector;
+
+/**
+ * ObjectInspector for Iceberg V3 unknown type columns.
+ * <p>
+ * Unknown values are always null and are not stored in data files.
+ */
+public final class UnknownObjectInspector implements ObjectInspector {
+
+ private static final ObjectInspector INSTANCE = new UnknownObjectInspector();
+
+ private UnknownObjectInspector() {
+ }
+
+ public static ObjectInspector get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.UNKNOWN;
+ }
+
+ @Override
+ public String getTypeName() {
+ return "unknown";
+ }
+}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java
b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java
index ca5abdbbf2c..6773b868700 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/thrift/Type.java
@@ -109,7 +109,10 @@ public enum Type {
USER_DEFINED_TYPE("USER_DEFINED",
java.sql.Types.OTHER,
TTypeId.USER_DEFINED_TYPE,
- true, false);
+ true, false),
+ UNKNOWN_TYPE(serdeConstants.UNKNOWN_TYPE_NAME.toUpperCase(),
+ java.sql.Types.NULL,
+ TTypeId.UNKNOWN_TYPE);
private final String name;
private final TTypeId tType;
@@ -262,6 +265,9 @@ public static Type getType(TypeInfo typeInfo) {
case UNION: {
return Type.UNION_TYPE;
}
+ case UNKNOWN: {
+ return Type.UNKNOWN_TYPE;
+ }
default: {
throw new RuntimeException("Unrecognized type: " +
typeInfo.getCategory());
}
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java
b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java
index f80c5192604..bee5ac82dc9 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoFactory.java
@@ -281,6 +281,10 @@ public static TypeInfo getVariantTypeInfo() {
return VariantTypeInfo.get();
}
+ public static TypeInfo getUnknownTypeInfo() {
+ return UnknownTypeInfo.get();
+ }
+
static ConcurrentHashMap<TypeInfo, TypeInfo> cachedListTypeInfo = new
ConcurrentHashMap<TypeInfo, TypeInfo>();
public static TypeInfo getListTypeInfo(TypeInfo elementTypeInfo) {
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
index b25d35868df..ef4bdd2ae7f 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/TypeInfoUtils.java
@@ -400,6 +400,7 @@ private Token expect(String item, String alternative) {
&& !serdeConstants.STRUCT_TYPE_NAME.equals(t.text)
&& !serdeConstants.UNION_TYPE_NAME.equals(t.text)
&& !serdeConstants.VARIANT_TYPE_NAME.equals(t.text)
+ && !serdeConstants.UNKNOWN_TYPE_NAME.equals(t.text)
&& null == PrimitiveObjectInspectorUtils
.getTypeEntryFromTypeName(t.text)
&& !t.text.equals(alternative)) {
@@ -579,6 +580,11 @@ private TypeInfo parseType() {
return TypeInfoFactory.getVariantTypeInfo();
}
+ // Is this an unknown type?
+ if (serdeConstants.UNKNOWN_TYPE_NAME.equals(t.text)) {
+ return TypeInfoFactory.getUnknownTypeInfo();
+ }
+
throw new RuntimeException("Internal error parsing position "
+ t.position + " of '" + typeInfoString + "'");
}
@@ -683,6 +689,14 @@ public static ObjectInspector
getStandardWritableObjectInspectorFromTypeInfo(
fieldObjectInspectors);
break;
}
+ case VARIANT: {
+ result = ObjectInspectorFactory.getVariantObjectInspector();
+ break;
+ }
+ case UNKNOWN: {
+ result = ObjectInspectorFactory.getUnknownObjectInspector();
+ break;
+ }
default: {
result = null;
@@ -765,6 +779,14 @@ public static ObjectInspector
getStandardJavaObjectInspectorFromTypeInfo(
fieldObjectInspectors);
break;
}
+ case VARIANT: {
+ result = ObjectInspectorFactory.getVariantObjectInspector();
+ break;
+ }
+ case UNKNOWN: {
+ result = ObjectInspectorFactory.getUnknownObjectInspector();
+ break;
+ }
default: {
result = null;
}
@@ -837,6 +859,14 @@ public static TypeInfo
getTypeInfoFromObjectInspector(ObjectInspector oi) {
result = TypeInfoFactory.getUnionTypeInfo(objectTypeInfos);
break;
}
+ case VARIANT: {
+ result = TypeInfoFactory.getVariantTypeInfo();
+ break;
+ }
+ case UNKNOWN: {
+ result = TypeInfoFactory.getUnknownTypeInfo();
+ break;
+ }
default: {
throw new RuntimeException("Unknown ObjectInspector category!");
}
@@ -1013,6 +1043,16 @@ public static boolean
implicitConvertible(PrimitiveCategory from, PrimitiveCateg
return true;
}
+ /**
+ * Returns whether a NULL literal (void) can be assigned to the target type.
+ * Iceberg unknown columns only accept NULL values.
+ */
+ public static boolean isVoidCompatibleTarget(TypeInfo from, TypeInfo to) {
+ return from.getCategory() == Category.PRIMITIVE
+ && ((PrimitiveTypeInfo) from).getPrimitiveCategory() ==
PrimitiveCategory.VOID
+ && to.getCategory() == Category.UNKNOWN;
+ }
+
/**
* Returns whether it is possible to implicitly convert an object of Class
* from to Class to.
@@ -1022,6 +1062,10 @@ public static boolean implicitConvertible(TypeInfo from,
TypeInfo to) {
return true;
}
+ if (isVoidCompatibleTarget(from, to)) {
+ return true;
+ }
+
// Reimplemented to use PrimitiveCategory rather than TypeInfo, because
// 2 TypeInfos from the same qualified type (varchar, decimal) should
still be
// seen as equivalent.
diff --git
a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java
b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java
new file mode 100644
index 00000000000..19c19524de4
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/UnknownTypeInfo.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.typeinfo;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.hadoop.hive.common.classification.InterfaceAudience;
+import org.apache.hadoop.hive.common.classification.InterfaceStability;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+
+/**
+ * UnknownTypeInfo represents the TypeInfo of an Iceberg V3 unknown type in
Hive.
+ * <p>
+ * Unknown columns are schema placeholders for undetermined types. They must
be optional,
+ * default to null, and are not stored in data files.
+ */
[email protected]
[email protected]
+public final class UnknownTypeInfo extends TypeInfo implements Serializable {
+
+ @Serial
+ private static final long serialVersionUID = 1L;
+
+ private static final UnknownTypeInfo INSTANCE = new UnknownTypeInfo();
+
+ public static UnknownTypeInfo get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public Category getCategory() {
+ return Category.UNKNOWN;
+ }
+
+ @Override
+ public String getTypeName() {
+ return "unknown";
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ return o instanceof UnknownTypeInfo;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(UnknownTypeInfo.class, getTypeName());
+ }
+}
diff --git
a/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java
b/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java
new file mode 100644
index 00000000000..87a73c2031e
--- /dev/null
+++
b/serde/src/test/org/apache/hadoop/hive/serde2/typeinfo/TestUnknownTypeInfo.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.typeinfo;
+
+import java.util.List;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.UnknownObjectInspector;
+import org.apache.hadoop.hive.serde2.thrift.Type;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestUnknownTypeInfo {
+
+ @Test
+ public void testParseUnknownType() {
+ TypeInfo typeInfo =
TypeInfoUtils.getTypeInfoFromTypeString(serdeConstants.UNKNOWN_TYPE_NAME);
+ Assert.assertEquals(UnknownTypeInfo.get(), typeInfo);
+ Assert.assertEquals(ObjectInspector.Category.UNKNOWN,
typeInfo.getCategory());
+ }
+
+ @Test
+ public void testUnknownObjectInspector() {
+ ObjectInspector oi = ObjectInspectorFactory.getUnknownObjectInspector();
+ Assert.assertEquals(UnknownObjectInspector.get(), oi);
+ Assert.assertEquals(ObjectInspector.Category.UNKNOWN, oi.getCategory());
+ Assert.assertEquals(serdeConstants.UNKNOWN_TYPE_NAME, oi.getTypeName());
+ }
+
+ @Test
+ public void testVoidIsCompatibleWithUnknownType() {
+ Assert.assertTrue(TypeInfoUtils.isVoidCompatibleTarget(
+ TypeInfoFactory.voidTypeInfo, TypeInfoFactory.getUnknownTypeInfo()));
+ Assert.assertTrue(TypeInfoUtils.implicitConvertible(
+ TypeInfoFactory.voidTypeInfo, TypeInfoFactory.getUnknownTypeInfo()));
+ }
+
+ @Test
+ public void testLazyObjectInspectorForUnknownType() throws SerDeException {
+ ObjectInspector oi = LazyFactory.createLazyStructInspector(
+ List.of("placeholder"),
+ List.of(TypeInfoFactory.getUnknownTypeInfo()),
+ new byte[] {(byte) 1, (byte) 2, (byte) 3, (byte) 4, (byte) 5, (byte)
6, (byte) 7, (byte) 8},
+ new Text("\\N"), false, false, (byte) 0);
+ Assert.assertNotNull(oi);
+ }
+
+ @Test
+ public void testStandardObjectInspectorFromTypeInfo() {
+ ObjectInspector oi =
TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(UnknownTypeInfo.get());
+ Assert.assertEquals(UnknownObjectInspector.get(), oi);
+ }
+
+ @Test
+ public void testThriftTypeMapping() {
+ Assert.assertEquals(Type.UNKNOWN_TYPE,
Type.getType(serdeConstants.UNKNOWN_TYPE_NAME));
+ Assert.assertEquals(Type.UNKNOWN_TYPE,
Type.getType(UnknownTypeInfo.get()));
+ }
+}
diff --git a/service-rpc/if/TCLIService.thrift
b/service-rpc/if/TCLIService.thrift
index a399e66445c..ee58cd9979d 100644
--- a/service-rpc/if/TCLIService.thrift
+++ b/service-rpc/if/TCLIService.thrift
@@ -94,7 +94,8 @@ enum TTypeId {
CHAR_TYPE,
INTERVAL_YEAR_MONTH_TYPE,
INTERVAL_DAY_TIME_TYPE,
- TIMESTAMPLOCALTZ_TYPE
+ TIMESTAMPLOCALTZ_TYPE,
+ UNKNOWN_TYPE
}
const set<TTypeId> PRIMITIVE_TYPES = [
@@ -115,7 +116,8 @@ const set<TTypeId> PRIMITIVE_TYPES = [
TTypeId.CHAR_TYPE,
TTypeId.INTERVAL_YEAR_MONTH_TYPE,
TTypeId.INTERVAL_DAY_TIME_TYPE,
- TTypeId.TIMESTAMPLOCALTZ_TYPE
+ TTypeId.TIMESTAMPLOCALTZ_TYPE,
+ TTypeId.UNKNOWN_TYPE
]
const set<TTypeId> COMPLEX_TYPES = [
@@ -154,6 +156,7 @@ const map<TTypeId,string> TYPE_NAMES = {
TTypeId.INTERVAL_YEAR_MONTH_TYPE: "INTERVAL_YEAR_MONTH"
TTypeId.INTERVAL_DAY_TIME_TYPE: "INTERVAL_DAY_TIME"
TTypeId.TIMESTAMPLOCALTZ_TYPE: "TIMESTAMP WITH LOCAL TIME ZONE"
+ TTypeId.UNKNOWN_TYPE: "UNKNOWN"
}
// Thrift does not support recursively defined types or forward declarations,
diff --git
a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
index 7b9c26b3d75..e525f2cdae3 100644
---
a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
+++
b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TCLIServiceConstants.java
@@ -29,6 +29,7 @@
PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_YEAR_MONTH_TYPE);
PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.INTERVAL_DAY_TIME_TYPE);
PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMPLOCALTZ_TYPE);
+
PRIMITIVE_TYPES.add(org.apache.hive.service.rpc.thrift.TTypeId.UNKNOWN_TYPE);
}
public static final java.util.Set<TTypeId> COMPLEX_TYPES =
java.util.EnumSet.noneOf(TTypeId.class);
@@ -69,6 +70,7 @@
TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TIMESTAMP_TYPE,
"TIMESTAMP");
TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.TINYINT_TYPE,
"TINYINT");
TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.UNION_TYPE,
"UNIONTYPE");
+ TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.UNKNOWN_TYPE,
"UNKNOWN");
TYPE_NAMES.put(org.apache.hive.service.rpc.thrift.TTypeId.VARCHAR_TYPE,
"VARCHAR");
}
diff --git
a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java
b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java
index 5039ce50b2e..57108255c1c 100644
---
a/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java
+++
b/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TTypeId.java
@@ -31,7 +31,8 @@ public enum TTypeId implements org.apache.thrift.TEnum {
CHAR_TYPE(19),
INTERVAL_YEAR_MONTH_TYPE(20),
INTERVAL_DAY_TIME_TYPE(21),
- TIMESTAMPLOCALTZ_TYPE(22);
+ TIMESTAMPLOCALTZ_TYPE(22),
+ UNKNOWN_TYPE(23);
private final int value;
@@ -99,6 +100,8 @@ public static TTypeId findByValue(int value) {
return INTERVAL_DAY_TIME_TYPE;
case 22:
return TIMESTAMPLOCALTZ_TYPE;
+ case 23:
+ return UNKNOWN_TYPE;
default:
return null;
}
diff --git a/service-rpc/src/gen/thrift/gen-php/TTypeId.php
b/service-rpc/src/gen/thrift/gen-php/TTypeId.php
index 43f773d2e1e..ac799c4a173 100644
--- a/service-rpc/src/gen/thrift/gen-php/TTypeId.php
+++ b/service-rpc/src/gen/thrift/gen-php/TTypeId.php
@@ -62,6 +62,8 @@ final class TTypeId
const TIMESTAMPLOCALTZ_TYPE = 22;
+ const UNKNOWN_TYPE = 23;
+
static public $__names = array(
0 => 'BOOLEAN_TYPE',
1 => 'TINYINT_TYPE',
@@ -86,6 +88,7 @@ final class TTypeId
20 => 'INTERVAL_YEAR_MONTH_TYPE',
21 => 'INTERVAL_DAY_TIME_TYPE',
22 => 'TIMESTAMPLOCALTZ_TYPE',
+ 23 => 'UNKNOWN_TYPE',
);
}
diff --git a/service/src/java/org/apache/hive/service/cli/ColumnValue.java
b/service/src/java/org/apache/hive/service/cli/ColumnValue.java
index 09ca12787e8..b994d6fc846 100644
--- a/service/src/java/org/apache/hive/service/cli/ColumnValue.java
+++ b/service/src/java/org/apache/hive/service/cli/ColumnValue.java
@@ -219,6 +219,8 @@ public static TColumnValue toTColumnValue(TypeDescriptor
typeDescriptor, Object
return stringValue((String)value);
case NULL_TYPE:
return stringValue((String)value);
+ case UNKNOWN_TYPE:
+ return stringValue((String)value);
default:
return null;
}
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
index 9db91218242..511cfc050b7 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/ColumnType.java
@@ -83,6 +83,7 @@ public class ColumnType {
public static final String UNION_TYPE_NAME = "uniontype";
public static final String VARIANT_TYPE_NAME = "variant";
+ public static final String UNKNOWN_TYPE_NAME = "unknown";
public static final String LIST_COLUMNS = "columns";
@@ -200,6 +201,7 @@ public class ColumnType {
STRUCT_TYPE_NAME,
UNION_TYPE_NAME,
VARIANT_TYPE_NAME,
+ UNKNOWN_TYPE_NAME,
LIST_COLUMNS,
LIST_COLUMN_TYPES,
COLUMN_NAME_DELIMITER
