This is an automated email from the ASF dual-hosted git repository.
lpinter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8ec1d4c989 HIVE-26282: Improve iceberg CTAS error message for
unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary)
8ec1d4c989 is described below
commit 8ec1d4c989baa0c4863084297b73e80e5929044d
Author: László Pintér <[email protected]>
AuthorDate: Fri Jun 3 13:43:10 2022 +0200
HIVE-26282: Improve iceberg CTAS error message for unsupported types
(#3337) (Laszlo Pinter, reviewed by Peter Vary)
---
.../apache/iceberg/hive/HiveSchemaConverter.java | 6 ++-
.../iceberg/mr/hive/TestHiveIcebergCTAS.java | 57 ++++++++++++++++++++++
2 files changed, 61 insertions(+), 2 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index 422546c112..9122577d41 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -83,7 +83,8 @@ class HiveSchemaConverter {
return Types.BooleanType.get();
case BYTE:
case SHORT:
- Preconditions.checkArgument(autoConvert, "Unsupported Hive type:
%s, use integer instead",
+ Preconditions.checkArgument(autoConvert, "Unsupported Hive type:
%s, use integer " +
+ "instead or enable automatic type conversion, set
'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
LOG.debug("Using auto conversion from SHORT/BYTE to INTEGER");
@@ -96,7 +97,8 @@ class HiveSchemaConverter {
return Types.BinaryType.get();
case CHAR:
case VARCHAR:
- Preconditions.checkArgument(autoConvert, "Unsupported Hive type:
%s, use string instead",
+ Preconditions.checkArgument(autoConvert, "Unsupported Hive type:
%s, use integer " +
+ "instead or enable automatic type conversion, set
'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
LOG.debug("Using auto conversion from CHAR/VARCHAR to STRING");
diff --git
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
index e41a4c6fc6..fda9652e0f 100644
---
a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
+++
b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.mr.hive;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.PartitionSpec;
@@ -32,8 +33,10 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.TestHelper;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.Assert;
@@ -222,4 +225,58 @@ public class TestHiveIcebergCTAS extends
HiveIcebergStorageHandlerWithEngineBase
objects = shell.executeStatement("SELECT * FROM target");
Assert.assertTrue(objects.isEmpty());
}
+
+ @Test
+ public void testCTASUnsupportedTypeWithoutAutoConversion() {
+ Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType ==
TestTables.TestTableType.HIVE_CATALOG);
+ Map<String, Type> notSupportedTypes = ImmutableMap.of(
+ "TINYINT", Types.IntegerType.get(),
+ "SMALLINT", Types.IntegerType.get(),
+ "VARCHAR(1)", Types.StringType.get(),
+ "CHAR(1)", Types.StringType.get());
+
+
+ for (String notSupportedType : notSupportedTypes.keySet()) {
+ shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED
AS ORC", notSupportedType));
+ AssertHelpers.assertThrows("should throw exception",
IllegalArgumentException.class,
+ "Unsupported Hive type: ", () -> {
+ shell.executeStatement(String.format(
+ "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM
source",
+
testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")),
+ testTables.propertiesForCreateTableSQL(
+ ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT,
fileFormat.toString())
+ )
+ ));
+ });
+ shell.executeStatement("DROP TABLE source");
+ }
+ }
+
+ @Test
+ public void testCTASUnsupportedTypeWithAutoConversion() {
+ Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType ==
TestTables.TestTableType.HIVE_CATALOG);
+ Map<String, Type> notSupportedTypes = ImmutableMap.of(
+ "TINYINT", Types.IntegerType.get(),
+ "SMALLINT", Types.IntegerType.get(),
+ "VARCHAR(1)", Types.StringType.get(),
+ "CHAR(1)", Types.StringType.get());
+
+ shell.setHiveSessionValue(InputFormatConfig.SCHEMA_AUTO_CONVERSION,
"true");
+
+ for (String notSupportedType : notSupportedTypes.keySet()) {
+ shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED
AS ORC", notSupportedType));
+ shell.executeStatement(String.format(
+ "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM
source",
+ testTables.locationForCreateTableSQL(TableIdentifier.of("default",
"target")),
+ testTables.propertiesForCreateTableSQL(
+ ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT,
fileFormat.toString())
+ )
+ ));
+
+ org.apache.iceberg.Table icebergTable =
testTables.loadTable(TableIdentifier.of("default", "target"));
+ Assert.assertEquals(notSupportedTypes.get(notSupportedType),
icebergTable.schema().columns().get(0).type());
+ shell.executeStatement("DROP TABLE source");
+ shell.executeStatement("DROP TABLE target");
+ }
+ }
}
