This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 4641f6e HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive
listing in HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163)
4641f6e is described below
commit 4641f6e484915cb8dfa5abedfc4732b68c88a758
Author: pvary <[email protected]>
AuthorDate: Fri Apr 1 13:51:27 2022 +0200
HIVE-26101: Port Iceberg Hive fix - Hive: Avoid recursive listing in
HiveCatalog#renameTable (Peter Vary reviewed by Marton Bod) (#3163)
---
.../java/org/apache/iceberg/hive/HiveCatalog.java | 2 +-
.../apache/iceberg/hive/HiveTableOperations.java | 16 ++-----------
.../org/apache/iceberg/hive/MetastoreUtil.java | 27 ++++++++++++++++++++++
3 files changed, 30 insertions(+), 15 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
index 880d60d..4737dd6 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveCatalog.java
@@ -214,7 +214,7 @@ public class HiveCatalog extends BaseMetastoreCatalog
implements SupportsNamespa
table.setTableName(to.name());
clients.run(client -> {
- client.alter_table(fromDatabase, fromName, table);
+ MetastoreUtil.alterTable(client, fromDatabase, fromName, table);
return null;
});
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
index ccde59c..3ad450c 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveTableOperations.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockLevel;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -57,7 +56,6 @@ import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotSummary;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableProperties;
-import org.apache.iceberg.common.DynMethods;
import org.apache.iceberg.exceptions.AlreadyExistsException;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
@@ -93,14 +91,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
private static final long HIVE_LOCK_CHECK_MIN_WAIT_MS_DEFAULT = 50; // 50
milliseconds
private static final long HIVE_LOCK_CHECK_MAX_WAIT_MS_DEFAULT = 5 * 1000; //
5 seconds
private static final long HIVE_TABLE_LEVEL_LOCK_EVICT_MS_DEFAULT =
TimeUnit.MINUTES.toMillis(10);
- private static final DynMethods.UnboundMethod ALTER_TABLE =
DynMethods.builder("alter_table")
- .impl(IMetaStoreClient.class, "alter_table_with_environmentContext",
- String.class, String.class, Table.class, EnvironmentContext.class)
- .impl(IMetaStoreClient.class, "alter_table",
- String.class, String.class, Table.class, EnvironmentContext.class)
- .impl(IMetaStoreClient.class, "alter_table",
- String.class, String.class, Table.class)
- .build();
+
private static final BiMap<String, String> ICEBERG_TO_HMS_TRANSLATION =
ImmutableBiMap.of(
// gc.enabled in Iceberg and external.table.purge in Hive are meant to
do the same things but with different names
GC_ENABLED, "external.table.purge"
@@ -310,10 +301,7 @@ public class HiveTableOperations extends
BaseMetastoreTableOperations {
void persistTable(Table hmsTable, boolean updateHiveTable) throws
TException, InterruptedException {
if (updateHiveTable) {
metaClients.run(client -> {
- EnvironmentContext envContext = new EnvironmentContext(
- ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS,
StatsSetupConst.TRUE)
- );
- ALTER_TABLE.invoke(client, database, tableName, hmsTable, envContext);
+ MetastoreUtil.alterTable(client, database, tableName, hmsTable);
return null;
});
} else {
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
index ad0ec80..76363f1 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java
@@ -19,12 +19,28 @@
package org.apache.iceberg.hive;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
+import org.apache.hadoop.hive.metastore.api.Table;
+import org.apache.iceberg.common.DynMethods;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+
public class MetastoreUtil {
// this class is unique to Hive3 and cannot be found in Hive2, therefore a
good proxy to see if
// we are working against Hive3 dependencies
private static final String HIVE3_UNIQUE_CLASS =
"org.apache.hadoop.hive.serde2.io.DateWritableV2";
+ private static final DynMethods.UnboundMethod ALTER_TABLE =
DynMethods.builder("alter_table")
+ .impl(IMetaStoreClient.class, "alter_table_with_environmentContext",
+ String.class, String.class, Table.class, EnvironmentContext.class)
+ .impl(IMetaStoreClient.class, "alter_table",
+ String.class, String.class, Table.class, EnvironmentContext.class)
+ .impl(IMetaStoreClient.class, "alter_table",
+ String.class, String.class, Table.class)
+ .build();
+
private static final boolean HIVE3_PRESENT_ON_CLASSPATH = detectHive3();
private MetastoreUtil() {
@@ -37,6 +53,17 @@ public class MetastoreUtil {
return HIVE3_PRESENT_ON_CLASSPATH;
}
+ /**
+ * Calls alter_table method using the metastore client. If possible, an
environmental context will be used that
+ * turns off stats updates to avoid recursive listing.
+ */
+ public static void alterTable(IMetaStoreClient client, String databaseName,
String tblName, Table table) {
+ EnvironmentContext envContext = new EnvironmentContext(
+ ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS,
StatsSetupConst.TRUE)
+ );
+ ALTER_TABLE.invoke(client, databaseName, tblName, table, envContext);
+ }
+
private static boolean detectHive3() {
try {
Class.forName(HIVE3_UNIQUE_CLASS);
