This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8ad1a2a72b1 HIVE-28817: Rebuilding a materialized view stored in
Iceberg fails when schema has varchar column - ADDENDUM (Krisztian Kasa,
reviewed by Denys Kuzmenko)
8ad1a2a72b1 is described below
commit 8ad1a2a72b16307bfd50763928b4a16ef92b72be
Author: Krisztian Kasa <[email protected]>
AuthorDate: Wed Mar 26 05:55:38 2025 +0100
HIVE-28817: Rebuilding a materialized view stored in Iceberg fails when
schema has varchar column - ADDENDUM (Krisztian Kasa, reviewed by Denys
Kuzmenko)
---
.../org/apache/iceberg/hive/MetastoreLock.java | 29 +++++++++++-----------
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 1 -
.../mr/hive/HiveIcebergOutputCommitter.java | 2 --
.../iceberg/mr/hive/HiveIcebergStorageHandler.java | 6 +++++
.../apache/iceberg/mr/hive/IcebergAcidUtil.java | 15 -----------
.../AlterMaterializedViewRebuildAnalyzer.java | 3 ++-
6 files changed, 23 insertions(+), 33 deletions(-)
diff --git
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
index 6ebf02d1400..c0d9d88ee9f 100644
---
a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
+++
b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreLock.java
@@ -21,6 +21,8 @@
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.Executors;
@@ -31,8 +33,6 @@
import java.util.concurrent.locks.ReentrantLock;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.LockRequestBuilder;
-import org.apache.hadoop.hive.metastore.api.DataOperationType;
import org.apache.hadoop.hive.metastore.api.LockComponent;
import org.apache.hadoop.hive.metastore.api.LockLevel;
import org.apache.hadoop.hive.metastore.api.LockRequest;
@@ -42,12 +42,12 @@
import org.apache.hadoop.hive.metastore.api.ShowLocksRequest;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponse;
import org.apache.hadoop.hive.metastore.api.ShowLocksResponseElement;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.iceberg.ClientPool;
import org.apache.iceberg.exceptions.CommitFailedException;
import
org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.MoreObjects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import
org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.iceberg.util.Tasks;
import org.apache.thrift.TException;
@@ -91,7 +91,6 @@ public class MetastoreLock implements HiveLock {
private final long lockHeartbeatIntervalTime;
private final ScheduledExecutorService exitingScheduledExecutorService;
private final String agentInfo;
- private final Configuration conf;
private Optional<Long> hmsLockId = Optional.empty();
private ReentrantLock jvmLock = null;
@@ -103,7 +102,6 @@ public MetastoreLock(Configuration conf,
ClientPool<IMetaStoreClient, TException
this.fullName = catalogName + "." + databaseName + "." + tableName;
this.databaseName = databaseName;
this.tableName = tableName;
- this.conf = conf;
this.lockAcquireTimeout =
conf.getLong(HIVE_ACQUIRE_LOCK_TIMEOUT_MS,
HIVE_ACQUIRE_LOCK_TIMEOUT_MS_DEFAULT);
@@ -270,18 +268,21 @@ private long acquireLock() throws LockException {
private LockInfo createLock() throws LockException {
LockInfo lockInfo = new LockInfo();
+ String hostName;
+ try {
+ hostName = InetAddress.getLocalHost().getHostName();
+ } catch (UnknownHostException uhe) {
+ throw new LockException(uhe, "Error generating host name");
+ }
+
LockComponent lockComponent =
new LockComponent(LockType.EXCL_WRITE, LockLevel.TABLE,
databaseName);
- lockComponent.setOperationType(DataOperationType.NO_TXN);
lockComponent.setTablename(tableName);
-
- // An open ACID transaction might exist when the SQL statement involves
both native and Iceberg tables.
- // Use it's txn id.
- LockRequest lockRequest = new LockRequestBuilder(null)
- .setTransactionId(conf.getLong(hive_metastoreConstants.TXN_ID, 0L))
- .setUser(HiveHadoopUtil.currentUser())
- .addLockComponent(lockComponent)
- .build();
+ LockRequest lockRequest =
+ new LockRequest(
+ Lists.newArrayList(lockComponent),
+ HiveHadoopUtil.currentUser(),
+ hostName);
// Only works in Hive 2 or later.
if (HiveVersion.min(HiveVersion.HIVE_2)) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index 17e0cdb285c..534afe11afe 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -414,7 +414,6 @@ public void
preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, E
}
try {
- conf.setLong(hive_metastoreConstants.TXN_ID, IcebergAcidUtil.getTxnId());
commitLock.lock();
doPreAlterTable(hmsTable, context);
} catch (Exception e) {
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
index 4d5dc4d1ef4..ae9d2c75b5a 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergOutputCommitter.java
@@ -43,7 +43,6 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
-import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context.Operation;
import org.apache.hadoop.hive.ql.Context.RewritePolicy;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -457,7 +456,6 @@ private void commitTable(FileIO io, ExecutorService
executor, OutputTable output
for (JobContext jobContext : jobContexts) {
JobConf conf = jobContext.getJobConf();
- conf.setLong(hive_metastoreConstants.TXN_ID, IcebergAcidUtil.getTxnId());
table = Optional.ofNullable(table).orElse(Catalogs.loadTable(conf,
catalogProperties));
branchName = conf.get(InputFormatConfig.OUTPUT_TABLE_SNAPSHOT_REF);
snapshotId = getSnapshotId(outputTable.table, branchName);
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
index df198242cd4..54ce20e3a58 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java
@@ -60,6 +60,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.AggrStats;
import org.apache.hadoop.hive.metastore.api.ColumnStatistics;
@@ -835,6 +836,11 @@ private void checkAndMergeColStats(List<ColumnStatistics>
statsNew, Table tbl) t
*/
@Override
public LockType getLockType(WriteEntity writeEntity) {
+ // Materialized views stored by Iceberg and the MV metadata is stored in
HMS doesn't need write locking because
+ // the locking is done by DbTxnManager.acquireMaterializationRebuildLock()
+ if (TableType.MATERIALIZED_VIEW == writeEntity.getTable().getTableType()) {
+ return LockType.SHARED_READ;
+ }
if (WriteEntity.WriteType.INSERT_OVERWRITE == writeEntity.getWriteType()) {
return LockType.EXCL_WRITE;
}
diff --git
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
index 0b71a9bca80..ae2220e8004 100644
---
a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
+++
b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/IcebergAcidUtil.java
@@ -28,9 +28,7 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.io.PositionDeleteInfo;
-import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.MetadataColumns;
import org.apache.iceberg.PartitionKey;
@@ -393,17 +391,4 @@ public T build() {
}
}
- static long getTxnId() {
- SessionState sessionState = SessionState.get();
- if (sessionState == null) {
- return 0L;
- }
-
- HiveTxnManager txnManager = sessionState.getTxnMgr();
- if (txnManager == null) {
- return 0L;
- }
-
- return txnManager.getCurrentTxnId();
- }
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
index f3f0769e114..542a1174ee2 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/materialized/alter/rebuild/AlterMaterializedViewRebuildAnalyzer.java
@@ -205,7 +205,8 @@ private ASTNode getRewrittenAST(TableName tableName) throws
SemanticException {
rewrittenAST = ParseUtils.parse(rewrittenInsertStatement, ctx);
this.ctx.addSubContext(ctx);
- if (!this.ctx.isExplainPlan() && AcidUtils.isTransactionalTable(table)) {
+ if (!this.ctx.isExplainPlan() && (AcidUtils.isTransactionalTable(table)
||
+ table.isNonNative() &&
table.getStorageHandler().areSnapshotsSupported())) {
// Acquire lock for the given materialized view. Only one rebuild per
materialized view can be triggered at a
// given time, as otherwise we might produce incorrect results if
incremental maintenance is triggered.
HiveTxnManager txnManager = getTxnMgr();
