This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9f180111e47 branch-3.1: [fix](hive) refresh hive cache when
inconsistency #58074 (#58570)
9f180111e47 is described below
commit 9f180111e4704dbbc87ae919f8a9975d2c3f863d
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Tue Dec 2 11:12:53 2025 +0800
branch-3.1: [fix](hive) refresh hive cache when inconsistency #58074
(#58570)
bp #58074
---
.../doris/datasource/hive/HMSExternalTable.java | 36 ++++++++++++++++------
.../apache/doris/datasource/hive/HiveDlaTable.java | 8 ++---
.../doris/datasource/hive/HiveMetaStoreCache.java | 6 +++-
.../doris/tablefunction/MetadataGenerator.java | 6 ++--
4 files changed, 36 insertions(+), 20 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index e38bb1fa3d6..e67a11a5b9d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -390,11 +390,8 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
if (CollectionUtils.isEmpty(this.getPartitionColumns())) {
return Collections.emptyMap();
}
- HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .getMetaStoreCache((HMSExternalCatalog) this.getCatalog());
- List<Type> partitionColumnTypes =
this.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(this));
- HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
- this, partitionColumnTypes);
+ HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
getHivePartitionValues(
+ MvccUtil.getSnapshotFromContext(this));
Map<Long, PartitionItem> idToPartitionItem =
hivePartitionValues.getIdToPartitionItem();
// transfer id to name
BiMap<Long, String> idToName =
hivePartitionValues.getPartitionNameToIdMap().inverse();
@@ -949,7 +946,6 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
return dlaTable.getTableSnapshot(context, snapshot);
}
-
@Override
public boolean isPartitionColumnAllowNull() {
makeSureInitialized();
@@ -1017,16 +1013,15 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
if (isView()) {
return null;
}
- HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .getMetaStoreCache((HMSExternalCatalog) catalog);
- List<Type> partitionColumnTypes =
getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(this));
+ Optional<MvccSnapshot> snapshot =
MvccUtil.getSnapshotFromContext(this);
+ List<Type> partitionColumnTypes = getPartitionColumnTypes(snapshot);
HiveMetaStoreCache.HivePartitionValues partitionValues = null;
// Get table partitions from cache.
if (!partitionColumnTypes.isEmpty()) {
// It is ok to get partition values from cache,
// no need to worry that this call will invalid or refresh the
cache.
// because it has enough space to keep partition info of all
tables in cache.
- partitionValues = cache.getPartitionValues(this,
partitionColumnTypes);
+ partitionValues = getHivePartitionValues(snapshot);
if (partitionValues == null ||
partitionValues.getPartitionNameToIdMap() == null) {
LOG.warn("Partition values for hive table {} is null", name);
} else {
@@ -1145,4 +1140,25 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
HMSCachedClient client = ((HMSExternalCatalog) catalog).getClient();
return client.getTable(getRemoteDbName(), remoteName);
}
+
+ public HiveMetaStoreCache.HivePartitionValues
getHivePartitionValues(Optional<MvccSnapshot> snapshot) {
+ HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
+ .getMetaStoreCache((HMSExternalCatalog) this.getCatalog());
+ try {
+ List<Type> partitionColumnTypes =
this.getPartitionColumnTypes(snapshot);
+ return cache.getPartitionValues(this, partitionColumnTypes);
+ } catch (Exception e) {
+ if
(e.getMessage().contains(HiveMetaStoreCache.ERR_CACHE_INCONSISTENCY)) {
+ LOG.warn("Hive metastore cache inconsistency detected for
table: {}.{}.{}. "
+ + "Clearing cache and retrying to get
partition values.",
+ this.getCatalog().getName(), this.getDbName(),
this.getName(), e);
+ ExternalSchemaCache schemaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().getSchemaCache(catalog);
+ schemaCache.invalidateTableCache(this);
+ List<Type> partitionColumnTypes =
this.getPartitionColumnTypes(snapshot);
+ return cache.getPartitionValues(this, partitionColumnTypes);
+ } else {
+ throw e;
+ }
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
index b517ca99983..d8969c42dec 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java
@@ -71,11 +71,10 @@ public class HiveDlaTable extends HMSDlaTable {
@Override
public MTMVSnapshotIf getPartitionSnapshot(String partitionName,
MTMVRefreshContext context,
Optional<MvccSnapshot> snapshot) throws AnalysisException {
+ HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
hmsTable.getHivePartitionValues(snapshot);
+ Long partitionId =
getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues);
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog());
- HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
- hmsTable, hmsTable.getPartitionColumnTypes(snapshot));
- Long partitionId =
getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues);
HivePartition hivePartition =
getHivePartitionByIdOrAnalysisException(partitionId,
hivePartitionValues, cache);
return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime());
@@ -90,10 +89,9 @@ public class HiveDlaTable extends HMSDlaTable {
HivePartition maxPartition = null;
long maxVersionTime = 0L;
long visibleVersionTime;
+ HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
hmsTable.getHivePartitionValues(snapshot);
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog());
- HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
- hmsTable, hmsTable.getPartitionColumnTypes(snapshot));
List<HivePartition> partitionList =
cache.getAllPartitionsWithCache(hmsTable,
Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values()));
if (CollectionUtils.isEmpty(partitionList)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 072de0cda94..883be276f6f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -107,6 +107,7 @@ public class HiveMetaStoreCache {
public static final String HIVE_DEFAULT_PARTITION =
"__HIVE_DEFAULT_PARTITION__";
// After hive 3, transactional table's will have file '_orc_acid_version'
with value >= '2'.
public static final String HIVE_ORC_ACID_VERSION_FILE =
"_orc_acid_version";
+ public static final String ERR_CACHE_INCONSISTENCY =
"ERR_CACHE_INCONSISTENCY: ";
private final HMSExternalCatalog catalog;
private JobConf jobConf;
@@ -285,11 +286,13 @@ public class HiveMetaStoreCache {
partitionNameToIdMap, idToUniqueIdsMap,
singleUidToColumnRangeMap, partitionValuesMap);
}
+ @VisibleForTesting
public ListPartitionItem toListPartitionItem(String partitionName,
List<Type> types) {
// Partition name will be in format: nation=cn/city=beijing
// parse it to get values "cn" and "beijing"
List<String> partitionValues =
HiveUtil.toPartitionValues(partitionName);
- Preconditions.checkState(partitionValues.size() == types.size(),
partitionName + " vs. " + types);
+ Preconditions.checkState(partitionValues.size() == types.size(),
+ ERR_CACHE_INCONSISTENCY + partitionName + " vs. " + types);
List<PartitionValue> values =
Lists.newArrayListWithExpectedSize(types.size());
for (String partitionValue : partitionValues) {
values.add(new PartitionValue(partitionValue,
HIVE_DEFAULT_PARTITION.equals(partitionValue)));
@@ -452,6 +455,7 @@ public class HiveMetaStoreCache {
return getPartitionValues(key);
}
+ @VisibleForTesting
public HivePartitionValues getPartitionValues(PartitionValueCacheKey key) {
return partitionValuesCache.get(key);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index 9430032b7bb..5a503e6ae1e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -1628,10 +1628,8 @@ public class MetadataGenerator {
"column " + colNames + " does not match partition columns
of table " + tbl.getName());
}
- HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
- .getMetaStoreCache((HMSExternalCatalog) tbl.getCatalog());
- HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
cache.getPartitionValues(
- tbl,
tbl.getPartitionColumnTypes(MvccUtil.getSnapshotFromContext(tbl)));
+ HiveMetaStoreCache.HivePartitionValues hivePartitionValues =
tbl.getHivePartitionValues(
+ MvccUtil.getSnapshotFromContext(tbl));
Map<Long, List<String>> valuesMap =
hivePartitionValues.getPartitionValuesMap();
List<TRow> dataBatch = Lists.newArrayList();
for (Map.Entry<Long, List<String>> entry : valuesMap.entrySet()) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
