This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 356b7e5ddf7868968fb76ca55a8046d0291388fd Author: Mihaly Szjatinya <[email protected]> AuthorDate: Fri Mar 7 22:30:22 2025 +0100 IMPALA-11597: Unset impala.lastComputeStatsTime during DROP STATS Removes 'impala.lastComputeStatsTime' property from table on DROP STATS catalog operation. Does not affect the incremental variant: 'DROP INCREMENTAL STATS t PARTITION (partition_spec)' Change-Id: Id743bc9779141df7a26a95a0ea201ef6fc6aeff9 Reviewed-on: http://gerrit.cloudera.org:8080/22474 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../apache/impala/service/CatalogOpExecutor.java | 49 +++++++++++++++++----- .../queries/QueryTest/iceberg-compute-stats.test | 15 +------ 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java index 8c7eb73d3..a71be6644 100644 --- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java +++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java @@ -2731,7 +2731,19 @@ public class CatalogOpExecutor { // TODO: Report the number of updated partitions/columns to the user? // TODO: bulk alter the partitions. dropColumnStats(table, catalogTimeline); - dropTableStats(table, catalogTimeline); + org.apache.hadoop.hive.metastore.api.Table msTbl = table.getMetaStoreTable(); + boolean isIntegratedIcebergTbl = + IcebergTable.isIcebergTable(msTbl) && isIcebergHmsIntegrationEnabled(msTbl); + if (isIntegratedIcebergTbl) { + org.apache.iceberg.Transaction iceTxn = + IcebergUtil.getIcebergTransaction((FeIcebergTable) table); + dropIntegratedIcebergTableStats(table, iceTxn); + IcebergCatalogOpExecutor.addCatalogVersionToTxn( + iceTxn, catalog_.getCatalogServiceId(), modification.newVersionNumber()); + iceTxn.commitTransaction(); + } else { + dropTableStats(table, catalogTimeline); + } } else { HdfsTable hdfsTbl = (HdfsTable) table; List<HdfsPartition> partitions = @@ -2810,22 +2822,20 @@ public class CatalogOpExecutor { private int dropTableStats(Table table, EventSequence catalogTimeline) throws ImpalaException { Preconditions.checkState(table.isWriteLockedByCurrentThread()); - // Delete the ROW_COUNT from the table (if it was set). org.apache.hadoop.hive.metastore.api.Table msTbl = table.getMetaStoreTable(); boolean isIntegratedIcebergTbl = IcebergTable.isIcebergTable(msTbl) && isIcebergHmsIntegrationEnabled(msTbl); - if (isIntegratedIcebergTbl) { - // We shouldn't modify table-level stats of HMS-integrated Iceberg tables as these - // stats are managed by Iceberg. - return 0; - } + Preconditions.checkState(!isIntegratedIcebergTbl); + // Delete the ROW_COUNT from the table (if it was set). int numTargetedPartitions = 0; boolean droppedRowCount = msTbl.getParameters().remove(StatsSetupConst.ROW_COUNT) != null; boolean droppedTotalSize = msTbl.getParameters().remove(StatsSetupConst.TOTAL_SIZE) != null; + boolean droppedLastCompute = + msTbl.getParameters().remove(HdfsTable.TBL_PROP_LAST_COMPUTE_STATS_TIME) != null; - if (droppedRowCount || droppedTotalSize) { + if (droppedRowCount || droppedTotalSize || droppedLastCompute) { applyAlterTable(msTbl, false, null, catalogTimeline); ++numTargetedPartitions; } @@ -2869,6 +2879,21 @@ public class CatalogOpExecutor { return modifiedParts.size(); } + /** + * For integrated Iceberg tables we shouldn't modify table-level stats of HMS-integrated + * Iceberg tables as these stats are managed by Iceberg. Dropping only + * "impala.lastComputeStatsTime" + */ + private void dropIntegratedIcebergTableStats( + Table table, org.apache.iceberg.Transaction iceTxn) throws ImpalaException { + org.apache.hadoop.hive.metastore.api.Table msTbl = table.getMetaStoreTable(); + boolean isIntegratedIcebergTbl = + IcebergTable.isIcebergTable(msTbl) && isIcebergHmsIntegrationEnabled(msTbl); + Preconditions.checkState(isIntegratedIcebergTbl); + IcebergCatalogOpExecutor.unsetTblProperties( + iceTxn, Collections.singletonList(HdfsTable.TBL_PROP_LAST_COMPUTE_STATS_TIME)); + } + /** * Drops a database from the metastore and removes the database's metadata from the * internal cache. Attempts to remove the HDFS cache directives of the underlying @@ -3521,14 +3546,18 @@ public class CatalogOpExecutor { modification.addCatalogServiceIdentifiersToTable(); try { FeIcebergTable iceTbl = (FeIcebergTable) table; + org.apache.iceberg.Transaction iceTxn = IcebergUtil.getIcebergTransaction(iceTbl); if (params.isDelete_stats()) { // TODO: The following methods could already generate events. Investigate if // calling modification.registerInflightEvent() here is necessary ahead of them. modification.registerInflightEvent(); dropColumnStats(table, catalogTimeline); - dropTableStats(table, catalogTimeline); + if (isIcebergHmsIntegrationEnabled(table.getMetaStoreTable())) { + dropIntegratedIcebergTableStats(table, iceTxn); + } else { + dropTableStats(table, catalogTimeline); + } } - org.apache.iceberg.Transaction iceTxn = IcebergUtil.getIcebergTransaction(iceTbl); IcebergCatalogOpExecutor.truncateTable(iceTxn); if (isIcebergHmsIntegrationEnabled(iceTbl.getMetaStoreTable())) { modification.registerInflightEvent(); diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-compute-stats.test index dfbee974c..5a4d96f3c 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-compute-stats.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-compute-stats.test @@ -75,13 +75,6 @@ drop stats ice_alltypes; STRING ==== ---- QUERY -alter table ice_alltypes unset tblproperties('impala.lastComputeStatsTime'); ----- RESULTS -'Updated table.' ----- TYPES -STRING -==== ----- QUERY # Table-level stats are not affected by DROP STATS. describe formatted ice_alltypes; ---- RESULTS: VERIFY_IS_SUBSET @@ -260,18 +253,12 @@ drop stats ice_alltypes_ht; STRING ==== ---- QUERY -alter table ice_alltypes_ht unset tblproperties('impala.lastComputeStatsTime'); ----- RESULTS -'Updated table.' ----- TYPES -STRING -==== ----- QUERY # Table-level stats are dropped for non-HMS integrated Iceberg tables. describe formatted ice_alltypes_ht; ---- RESULTS: VERIFY_IS_NOT_IN '','numRows ','2 ' row_regex:'','impala.lastComputeStatsTime','\d+\s+' +row_regex:'','totalSize ','\d+\s+' ---- TYPES STRING, STRING, STRING ====
