This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e0578fe620c [fix](iceberg) Invalidate related-table cache on external
table refresh (#60867)
e0578fe620c is described below
commit e0578fe620c54b556e98d627f24de847f6567531
Author: Socrates <[email protected]>
AuthorDate: Fri Feb 27 17:33:43 2026 +0800
[fix](iceberg) Invalidate related-table cache on external table refresh
(#60867)
Related PR: #57972
Problem Summary:
In multi-FE deployments, Iceberg partition evolution (ADD/REPLACE/DROP
PARTITION KEY) can leave some FEs with a stale
`IcebergExternalTable.isValidRelatedTableCached` value. This stale cache
may cause incorrect partitioned/unpartitioned
judgment and query failures until the table is refreshed.
This PR:
- Clears the related-table validation cache during
`OP_REFRESH_EXTERNAL_TABLE` replay
(`RefreshManager.refreshTableInternal()`).
- Removes the redundant per-operation manual cache clearing in
`IcebergMetadataOps`.
---
.../src/main/java/org/apache/doris/catalog/RefreshManager.java | 6 ++++++
.../org/apache/doris/datasource/iceberg/IcebergMetadataOps.java | 7 -------
2 files changed, 6 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 54237f2d38e..4ee6a2cab07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -29,6 +29,7 @@ import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HiveMetaStoreCache;
+import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.persist.OperationType;
import com.google.common.base.Strings;
@@ -232,6 +233,11 @@ public class RefreshManager {
public void refreshTableInternal(ExternalDatabase db, ExternalTable table,
long updateTime) {
table.unsetObjectCreated();
+ // Iceberg partition evolution can change partition specs across FEs.
+ // Clear related-table validation cache to avoid stale
partitioned/unpartitioned judgment.
+ if (table instanceof IcebergExternalTable) {
+ ((IcebergExternalTable) table).setIsValidRelatedTableCached(false);
+ }
if (updateTime > 0) {
table.setUpdateTime(updateTime);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 95b109ce3d5..bcdb7e2f39d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -859,8 +859,6 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
refreshTable(dorisTable, updateTime);
- // Reset cached isValidRelatedTable flag after partition evolution
- ((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
/**
@@ -888,8 +886,6 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
refreshTable(dorisTable, updateTime);
- // Reset cached isValidRelatedTable flag after partition evolution
- ((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
/**
@@ -931,8 +927,6 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
+ ", error message is: " +
ExceptionUtils.getRootCauseMessage(e), e);
}
refreshTable(dorisTable, updateTime);
- // Reset cached isValidRelatedTable flag after partition evolution
- ((IcebergExternalTable)
dorisTable).setIsValidRelatedTableCached(false);
}
@Override
@@ -1053,4 +1047,3 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
}
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
