This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 708d758dde1 HIVE-27441: Iceberg: CTLT with source table as Hive
managed table fails with "The table must be stored using an ACID compliant
format" (#4426). (Ayush Saxena, reviewed by Denys Kuzmenko, Attila Turoczy,
Simhadri Govindappa)
708d758dde1 is described below
commit 708d758dde10bf597f10a0692fe64c3e2506156f
Author: Ayush Saxena <[email protected]>
AuthorDate: Thu Jun 22 09:43:04 2023 +0530
HIVE-27441: Iceberg: CTLT with source table as Hive managed table fails
with "The table must be stored using an ACID compliant format" (#4426). (Ayush
Saxena, reviewed by Denys Kuzmenko, Attila Turoczy, Simhadri Govindappa)
---
.../src/test/queries/positive/ctlt_iceberg.q | 11 ++++++
.../src/test/results/positive/ctlt_iceberg.q.out | 39 ++++++++++++++++++++--
.../table/convert/AlterTableConvertOperation.java | 2 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 9 ++++-
4 files changed, 56 insertions(+), 5 deletions(-)
diff --git a/iceberg/iceberg-handler/src/test/queries/positive/ctlt_iceberg.q
b/iceberg/iceberg-handler/src/test/queries/positive/ctlt_iceberg.q
index 8d77812a729..895f51a4ff2 100644
--- a/iceberg/iceberg-handler/src/test/queries/positive/ctlt_iceberg.q
+++ b/iceberg/iceberg-handler/src/test/queries/positive/ctlt_iceberg.q
@@ -3,6 +3,8 @@
-- Mask random uuid
--! qt:replace:/(\s+'uuid'=')\S+('\s*)/$1#Masked#$2/
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
set hive.explain.user=false;
create table source(a int) stored by iceberg tblproperties
('format-version'='2') ;
@@ -45,3 +47,12 @@ create table emp_like2 like emp stored by iceberg;
-- Partition column should be there
show create table emp_like2;
+-- create a managed table
+create managed table man_table (id int) Stored as orc TBLPROPERTIES
('transactional'='true');
+
+create table like_man_table like man_table stored by iceberg;
+
+-- insert some data into the table and see if things work
+insert into like_man_table values (1), (2), (3), (4);
+
+select * from like_man_table order by id;
\ No newline at end of file
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out
b/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out
index d806a28de73..447a4b27f82 100644
--- a/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out
+++ b/iceberg/iceberg-handler/src/test/results/positive/ctlt_iceberg.q.out
@@ -38,7 +38,6 @@ STORED BY
LOCATION
'hdfs://### HDFS PATH ###'
TBLPROPERTIES (
- 'TRANSLATED_TO_EXTERNAL'='TRUE',
'bucketing_version'='2',
'created_with_ctlt'='true',
'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"a","required":false,"type":"int"}]}',
@@ -162,7 +161,6 @@ STORED BY
LOCATION
'hdfs://### HDFS PATH ###'
TBLPROPERTIES (
- 'TRANSLATED_TO_EXTERNAL'='TRUE',
'bucketing_version'='2',
'created_with_ctlt'='true',
'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"company","required":false,"type":"string"}]}',
@@ -230,7 +228,6 @@ STORED BY
LOCATION
'hdfs://### HDFS PATH ###'
TBLPROPERTIES (
- 'TRANSLATED_TO_EXTERNAL'='TRUE',
'bucketing_version'='2',
'created_with_ctlt'='true',
'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"id","required":false,"type":"int"},{"id":2,"name":"company","required":false,"type":"string"}]}',
@@ -242,3 +239,39 @@ TBLPROPERTIES (
'table_type'='ICEBERG',
#### A masked pattern was here ####
'uuid'='#Masked#')
+PREHOOK: query: create managed table man_table (id int) Stored as orc
TBLPROPERTIES ('transactional'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@man_table
+POSTHOOK: query: create managed table man_table (id int) Stored as orc
TBLPROPERTIES ('transactional'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@man_table
+PREHOOK: query: create table like_man_table like man_table stored by iceberg
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@like_man_table
+POSTHOOK: query: create table like_man_table like man_table stored by iceberg
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@like_man_table
+PREHOOK: query: insert into like_man_table values (1), (2), (3), (4)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@like_man_table
+POSTHOOK: query: insert into like_man_table values (1), (2), (3), (4)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@like_man_table
+PREHOOK: query: select * from like_man_table order by id
+PREHOOK: type: QUERY
+PREHOOK: Input: default@like_man_table
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: select * from like_man_table order by id
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@like_man_table
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1
+2
+3
+4
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/convert/AlterTableConvertOperation.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/convert/AlterTableConvertOperation.java
index 7a1e82a9fe3..4dcb9dc8d53 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/convert/AlterTableConvertOperation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/convert/AlterTableConvertOperation.java
@@ -38,7 +38,7 @@ import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE
*/
public class AlterTableConvertOperation extends
AbstractAlterTableOperation<AlterTableConvertDesc> {
- private enum ConversionFormats {
+ public enum ConversionFormats {
ICEBERG(ImmutableMap.of(META_TABLE_STORAGE,
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler")),
ACID(ImmutableMap.of(TABLE_IS_TRANSACTIONAL, "true",
TABLE_TRANSACTIONAL_PROPERTIES,
DEFAULT_TRANSACTIONAL_PROPERTY));
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 0e8d309210b..bbbc467ca76 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -119,6 +119,7 @@ import
org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
import org.apache.hadoop.hive.ql.ddl.DDLWork;
import org.apache.hadoop.hive.ql.ddl.misc.hooks.InsertCommitHookDesc;
import org.apache.hadoop.hive.ql.ddl.table.constraint.ConstraintsUtils;
+import org.apache.hadoop.hive.ql.ddl.table.convert.AlterTableConvertOperation;
import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableDesc;
import org.apache.hadoop.hive.ql.ddl.table.create.like.CreateTableLikeDesc;
import org.apache.hadoop.hive.ql.ddl.table.misc.preinsert.PreInsertTableDesc;
@@ -13967,7 +13968,8 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
isTransactional, isManaged, new String[]{qualifiedTabName.getDb(),
qualifiedTabName.getTable()}, isDefaultTableTypeChanged);
tblProps.put(hive_metastoreConstants.TABLE_IS_CTLT, "true");
- isExt = isExternalTableChanged(tblProps, isTransactional, isExt,
isDefaultTableTypeChanged);
+ isExt = isIcebergTable(tblProps) ||
+ isExternalTableChanged(tblProps, isTransactional, isExt,
isDefaultTableTypeChanged);
addDbAndTabToOutputs(new String[] {qualifiedTabName.getDb(),
qualifiedTabName.getTable()},
TableType.MANAGED_TABLE, isTemporary, tblProps, storageFormat);
@@ -14080,6 +14082,11 @@ public class SemanticAnalyzer extends
BaseSemanticAnalyzer {
return null;
}
+ private static boolean isIcebergTable(Map<String, String> tblProps) {
+ return
AlterTableConvertOperation.ConversionFormats.ICEBERG.properties().get(META_TABLE_STORAGE)
+ .equalsIgnoreCase(tblProps.get(META_TABLE_STORAGE));
+ }
+
private void validateStorageFormat(
StorageFormat storageFormat, Map<String, String> tblProps, boolean
partitionTransformSpecExists)
throws SemanticException {
