This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 3eb56ecae05 HIVE-28323: Iceberg: Allow reading tables irrespective
whether they were created with hive engined enabled or not. (#5293). (Ayush
Saxena, reviewed by Simhadri Govindappa)
3eb56ecae05 is described below
commit 3eb56ecae056d36370ff0ee9a0936c28b431724a
Author: Ayush Saxena <[email protected]>
AuthorDate: Fri Jun 14 12:26:38 2024 +0530
HIVE-28323: Iceberg: Allow reading tables irrespective whether they were
created with hive engined enabled or not. (#5293). (Ayush Saxena, reviewed by
Simhadri Govindappa)
---
.../apache/iceberg/hive/HiveTableOperations.java | 4 +-
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 10 ++++
.../queries/positive/show_create_iceberg_table.q | 7 +++
.../positive/show_create_iceberg_table.q.out | 61 ++++++++++++++++++++++
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 15 +++++-
.../apache/hadoop/hive/metastore/HiveMetaHook.java | 1 +
6 files changed, 94 insertions(+), 4 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index 84ad202375d..aa6c1232e62 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -85,7 +85,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
private static final String NO_LOCK_EXPECTED_VALUE =
"expected_parameter_value";
private static final long HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT = 32672;
- private static final String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+ public static final String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION =
ImmutableBiMap.of(
// gc.enabled in Iceberg and external.table.purge in Hive are meant to
do the same things but with different names
@@ -602,7 +602,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
* @param storageHandler Storage Handler class
* @return true if the storage_handler property is set to
HIVE_ICEBERG_STORAGE_HANDLER
*/
- private static boolean isHiveIcebergStorageHandler(String storageHandler) {
+ public static boolean isHiveIcebergStorageHandler(String storageHandler) {
try {
Class<?> storageHandlerClass = Class.forName(storageHandler);
Class<?> icebergStorageHandlerClass =
Class.forName(HIVE_ICEBERG_STORAGE_HANDLER);
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index b27ab64d512..84d9531b7ac 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -1071,6 +1071,16 @@ public class HiveIcebergMetaHook implements HiveMetaHook
{
if (!"1".equals(formatVersion)) {
hmsTable.getParameters().put(TableProperties.FORMAT_VERSION,
formatVersion);
}
+ // Set the serde info
+
hmsTable.getSd().setInputFormat(HiveIcebergInputFormat.class.getName());
+
hmsTable.getSd().setOutputFormat(HiveIcebergOutputFormat.class.getName());
+
hmsTable.getSd().getSerdeInfo().setSerializationLib(HiveIcebergSerDe.class.getName());
+ String storageHandler =
hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE);
+ // Check if META_TABLE_STORAGE is not present or is not an instance of
ICEBERG_STORAGE_HANDLER
+ if (storageHandler == null ||
!HiveTableOperations.isHiveIcebergStorageHandler(storageHandler)) {
+ hmsTable.getParameters()
+ .put(hive_metastoreConstants.META_TABLE_STORAGE,
HiveTableOperations.HIVE_ICEBERG_STORAGE_HANDLER);
+ }
} catch (NoSuchTableException | NotFoundException ex) {
// If the table doesn't exist, ignore throwing exception from here
}
diff --git
a/iceberg/iceberg-handler/src/test/queries/positive/show_create_iceberg_table.q
b/iceberg/iceberg-handler/src/test/queries/positive/show_create_iceberg_table.q
index ed1c63b639a..40414b51ccd 100644
---
a/iceberg/iceberg-handler/src/test/queries/positive/show_create_iceberg_table.q
+++
b/iceberg/iceberg-handler/src/test/queries/positive/show_create_iceberg_table.q
@@ -25,3 +25,10 @@ DROP TABLE IF EXISTS ice_data;
CREATE EXTERNAL TABLE ice_data (i int, s string) STORED BY ICEBERG;
INSERT INTO ice_data VALUES (1, 'ABC'),(2, 'CCC'),(3, 'DBD');
SHOW CREATE TABLE ice_data;
+
+set iceberg.engine.hive.enabled=false;
+DROP TABLE IF EXISTS ice_noHive;
+CREATE EXTERNAL TABLE ice_noHive (i int, s string) STORED BY ICEBERG;
+SHOW CREATE TABLE ice_noHive;
+INSERT INTO ice_noHive VALUES (1, 'ABC'),(2, 'CCC'),(3, 'DBD');
+SELECT * FROM ice_noHive;
diff --git
a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
index 3cc22bb42cf..72fc66b887d 100644
---
a/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
+++
b/iceberg/iceberg-handler/src/test/results/positive/show_create_iceberg_table.q.out
@@ -254,3 +254,64 @@ TBLPROPERTIES (
'table_type'='ICEBERG',
#### A masked pattern was here ####
'uuid'='#Masked#')
+PREHOOK: query: DROP TABLE IF EXISTS ice_noHive
+PREHOOK: type: DROPTABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: DROP TABLE IF EXISTS ice_noHive
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Output: database:default
+PREHOOK: query: CREATE EXTERNAL TABLE ice_noHive (i int, s string) STORED BY
ICEBERG
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@ice_noHive
+POSTHOOK: query: CREATE EXTERNAL TABLE ice_noHive (i int, s string) STORED BY
ICEBERG
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@ice_noHive
+PREHOOK: query: SHOW CREATE TABLE ice_noHive
+PREHOOK: type: SHOW_CREATETABLE
+PREHOOK: Input: default@ice_nohive
+POSTHOOK: query: SHOW CREATE TABLE ice_noHive
+POSTHOOK: type: SHOW_CREATETABLE
+POSTHOOK: Input: default@ice_nohive
+CREATE EXTERNAL TABLE `ice_nohive`(
+ `i` int,
+ `s` string)
+ROW FORMAT SERDE
+ 'org.apache.iceberg.mr.hive.HiveIcebergSerDe'
+STORED BY
+ 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
+
+LOCATION
+ 'hdfs://### HDFS PATH ###'
+TBLPROPERTIES (
+ 'bucketing_version'='2',
+
'current-schema'='{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"s","required":false,"type":"string"}]}',
+ 'format-version'='2',
+ 'iceberg.orc.files.only'='false',
+ 'metadata_location'='hdfs://### HDFS PATH ###',
+ 'parquet.compression'='zstd',
+ 'serialization.format'='1',
+ 'snapshot-count'='0',
+ 'table_type'='ICEBERG',
+#### A masked pattern was here ####
+ 'uuid'='#Masked#')
+PREHOOK: query: INSERT INTO ice_noHive VALUES (1, 'ABC'),(2, 'CCC'),(3, 'DBD')
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@ice_nohive
+POSTHOOK: query: INSERT INTO ice_noHive VALUES (1, 'ABC'),(2, 'CCC'),(3, 'DBD')
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@ice_nohive
+PREHOOK: query: SELECT * FROM ice_noHive
+PREHOOK: type: QUERY
+PREHOOK: Input: default@ice_nohive
+PREHOOK: Output: hdfs://### HDFS PATH ###
+POSTHOOK: query: SELECT * FROM ice_noHive
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@ice_nohive
+POSTHOOK: Output: hdfs://### HDFS PATH ###
+1 ABC
+2 CCC
+3 DBD
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 82132d2f17b..d3c27fb10be 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -33,6 +33,7 @@ import static
org.apache.hadoop.hive.common.AcidConstants.SOFT_DELETE_TABLE;
import static
org.apache.hadoop.hive.conf.Constants.MATERIALIZED_VIEW_REWRITING_TIME_WINDOW;
import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_LOAD_DYNAMIC_PARTITIONS_SCAN_SPECIFIC_PARTITIONS;
import static
org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_WRITE_NOTIFICATION_MAX_BATCH_SIZE;
+import static
org.apache.hadoop.hive.metastore.HiveMetaHook.HIVE_ICEBERG_STORAGE_HANDLER;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.CTAS_LEGACY_CONFIG;
import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
import static
org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.convertToGetPartitionsByNamesRequest;
@@ -5850,8 +5851,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
if (tbl == null) {
return null;
}
- HiveStorageHandler storageHandler =
- HiveUtils.getStorageHandler(conf,
tbl.getParameters().get(META_TABLE_STORAGE));
+ String className = getStorageHandlerClassName(tbl);
+ HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf,
className);
return storageHandler;
} catch (HiveException ex) {
LOG.error("Failed createStorageHandler", ex);
@@ -5860,6 +5861,16 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
+ private static String
getStorageHandlerClassName(org.apache.hadoop.hive.metastore.api.Table tbl) {
+ String tableType = tbl.getParameters().get(HiveMetaHook.TABLE_TYPE);
+ String metaTableStorage = tbl.getParameters().get(META_TABLE_STORAGE);
+ if (HiveMetaHook.ICEBERG.equalsIgnoreCase(tableType) && metaTableStorage
== null) {
+ return HIVE_ICEBERG_STORAGE_HANDLER;
+ }
+
+ return metaTableStorage;
+ }
+
public static class SchemaException extends MetaException {
private static final long serialVersionUID = 1L;
public SchemaException(String message) {
diff --git
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
index 115942b9b8f..11dfa1120c8 100644
---
a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
+++
b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaHook.java
@@ -57,6 +57,7 @@ public interface HiveMetaHook {
String TABLE_TYPE = "table_type";
String EXTERNAL = "EXTERNAL";
String ICEBERG = "ICEBERG";
+ String HIVE_ICEBERG_STORAGE_HANDLER =
"org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
String PROPERTIES_SEPARATOR = "'";
String MIGRATE_HIVE_TO_ICEBERG = "migrate_hive_to_iceberg";
