deniskuzZ commented on code in PR #5722:
URL: https://github.com/apache/hive/pull/5722#discussion_r2024462817
##########
iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java:
##########
@@ -130,6 +130,28 @@ public void testStatsWithInsert() {
checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 5);
}
+ @Test
+ public void testStatsWithPessimisticLockInsert() {
+ TableIdentifier identifier = TableIdentifier.of("default", "customers");
+
+ shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname,
true);
+
shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED.varname,
true);
+ testTables.createTable(shell, identifier.name(),
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of(), 1,
+ ImmutableMap.of(TableProperties.HIVE_LOCK_ENABLED, "false"));
Review Comment:
HiveIcebergStorageHandler
````
public LockType getLockType(WriteEntity writeEntity) {
org.apache.hadoop.hive.ql.metadata.Table hmsTable =
writeEntity.getTable();
// Materialized views stored by Iceberg and the MV metadata is stored in
HMS doesn't need write locking because
// the locking is done by
DbTxnManager.acquireMaterializationRebuildLock()
if (TableType.MATERIALIZED_VIEW == hmsTable.getTableType()) {
return LockType.SHARED_READ;
}
if (HiveTableOperations.hiveLockEnabled(hmsTable.getParameters(), conf))
{
throw new RuntimeException("Hive locking for table `" +
hmsTable.getFullTableName() +
"`cannot be enabled when `engine.hive.lock-enabled`=`true`. " +
"Disable `engine.hive.lock-enabled` to use Hive locking");
}
if (WriteEntity.WriteType.INSERT_OVERWRITE ==
writeEntity.getWriteType()) {
return LockType.EXCL_WRITE;
}
return LockType.SHARED_WRITE;
}
````
HiveTableOperations
````
private static boolean hiveLockEnabled(TableMetadata metadata,
Configuration conf) {
return hiveLockEnabled(metadata != null ? metadata.properties() : null,
conf);
}
public static boolean hiveLockEnabled(Map<String, String> properties,
Configuration conf) {
if (properties != null &&
properties.containsKey(TableProperties.HIVE_LOCK_ENABLED)) {
// We know that the property is set, so default value will not be used,
return PropertyUtil.propertyAsBoolean(properties,
TableProperties.HIVE_LOCK_ENABLED, false);
}
return conf.getBoolean(
ConfigProperties.LOCK_HIVE_ENABLED,
TableProperties.HIVE_LOCK_ENABLED_DEFAULT);
}
````
HiveIcebergMetaHook
````
private HiveLock lockObject(org.apache.hadoop.hive.metastore.api.Table
hmsTable) {
if (!HiveTableOperations.hiveLockEnabled(hmsTable.getParameters(), conf)
||
SessionStateUtil.getQueryState(conf)
.map(QueryState::getHiveOperation)
.filter(opType -> HiveOperation.QUERY == opType)
.isPresent()) {
return new NoLock();
} else {
return new MetastoreLock(
conf,
new CachedClientPool(conf, Maps.fromProperties(catalogProperties)),
catalogProperties.getProperty(Catalogs.NAME), hmsTable.getDbName(),
hmsTable.getTableName());
}
}
````
##########
iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStatistics.java:
##########
@@ -130,6 +130,28 @@ public void testStatsWithInsert() {
checkColStatMinMaxValue(identifier.name(), "customer_id", 0, 5);
}
+ @Test
+ public void testStatsWithPessimisticLockInsert() {
+ TableIdentifier identifier = TableIdentifier.of("default", "customers");
+
+ shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_STATS_AUTOGATHER.varname,
true);
+
shell.setHiveSessionValue(HiveConf.ConfVars.HIVE_TXN_EXT_LOCKING_ENABLED.varname,
true);
+ testTables.createTable(shell, identifier.name(),
HiveIcebergStorageHandlerTestUtils.CUSTOMER_SCHEMA,
+ PartitionSpec.unpartitioned(), fileFormat, ImmutableList.of(), 1,
+ ImmutableMap.of(TableProperties.HIVE_LOCK_ENABLED, "false"));
Review Comment:
you are right.
In that case I think we should add a fallback to
`HiveIcebergStorageHandler#getLockType` and when `HIVE_LOCK_ENABLED=true` throw
an exception "Hive locking can't be enabled when HIVE_LOCK_ENABLED=true" and
return SHARED_READ lock
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: gitbox-unsubscr...@hive.apache.org
For additional commands, e-mail: gitbox-h...@hive.apache.org
