This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dee2b70d3e4 [enhance](mtmv)hive cache add partitionId to partitionName
Map (#38525)
dee2b70d3e4 is described below
commit dee2b70d3e433b2c046198aefeb723440e9befcb
Author: zhangdong <[email protected]>
AuthorDate: Mon Aug 12 11:05:00 2024 +0800
[enhance](mtmv)hive cache add partitionId to partitionName Map (#38525)
MTMV need get partitionName by partitionId
but now only have partitionName=>partitionId Map,when there are many
partitions, obtaining the name from this map based on the ID can be
slow,
---
.../java/org/apache/doris/catalog/OlapTable.java | 10 ----------
.../doris/datasource/hive/HMSExternalTable.java | 20 +++-----------------
.../doris/datasource/hive/HiveMetaStoreCache.java | 10 ++++++----
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 9 ---------
.../org/apache/doris/mtmv/MTMVRelatedTableIf.java | 9 ---------
.../org/apache/doris/datasource/CatalogMgrTest.java | 5 ++++-
.../org/apache/doris/mtmv/MTMVPartitionUtilTest.java | 8 --------
7 files changed, 13 insertions(+), 58 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index a1fbbcc1fb1..884cd4f4054 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -2921,16 +2921,6 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
return new MTMVVersionSnapshot(visibleVersion);
}
- @Override
- public String getPartitionName(long partitionId) throws AnalysisException {
- readLock();
- try {
- return getPartitionOrAnalysisException(partitionId).getName();
- } finally {
- readUnlock();
- }
- }
-
private static Cloud.GetVersionResponse
getVersionFromMeta(Cloud.GetVersionRequest req)
throws RpcException {
long startAt = System.nanoTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
index 6752ae5dcbe..e1afbf3dc0b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java
@@ -48,6 +48,7 @@ import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
+import com.google.common.collect.BiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -732,28 +733,13 @@ public class HMSExternalTable extends ExternalTable
implements MTMVRelatedTableI
getDbName(), getName(), getPartitionColumnTypes());
Map<String, PartitionItem> res = Maps.newHashMap();
Map<Long, PartitionItem> idToPartitionItem =
hivePartitionValues.getIdToPartitionItem();
+ BiMap<Long, String> idToName =
hivePartitionValues.getPartitionNameToIdMap().inverse();
for (Entry<Long, PartitionItem> entry : idToPartitionItem.entrySet()) {
- try {
- res.put(getPartitionName(entry.getKey()), entry.getValue());
- } catch (AnalysisException e) {
- LOG.info("can not get partitionName by: " + entry.getKey());
- }
-
+ res.put(idToName.get(entry.getKey()), entry.getValue());
}
return res;
}
- @Override
- public String getPartitionName(long partitionId) throws AnalysisException {
- Map<String, Long> partitionNameToIdMap =
getHivePartitionValues().getPartitionNameToIdMap();
- for (Entry<String, Long> entry : partitionNameToIdMap.entrySet()) {
- if (entry.getValue().equals(partitionId)) {
- return entry.getKey();
- }
- }
- throw new AnalysisException("can not find partition, partitionId: " +
partitionId);
- }
-
private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() {
HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr()
.getMetaStoreCache((HMSExternalCatalog) getCatalog());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 631362a5b41..312f2382b0d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -54,6 +54,8 @@ import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -246,7 +248,7 @@ public class HiveMetaStoreCache {
LOG.debug("load #{} partitions for {} in catalog {}",
partitionNames.size(), key, catalog.getName());
}
Map<Long, PartitionItem> idToPartitionItem =
Maps.newHashMapWithExpectedSize(partitionNames.size());
- Map<String, Long> partitionNameToIdMap =
Maps.newHashMapWithExpectedSize(partitionNames.size());
+ BiMap<String, Long> partitionNameToIdMap =
HashBiMap.create(partitionNames.size());
Map<Long, List<UniqueId>> idToUniqueIdsMap =
Maps.newHashMapWithExpectedSize(partitionNames.size());
long idx = 0;
for (String partitionName : partitionNames) {
@@ -1074,7 +1076,7 @@ public class HiveMetaStoreCache {
@Data
public static class HivePartitionValues {
private long nextPartitionId;
- private Map<String, Long> partitionNameToIdMap;
+ private BiMap<String, Long> partitionNameToIdMap;
private Map<Long, List<UniqueId>> idToUniqueIdsMap;
private Map<Long, PartitionItem> idToPartitionItem;
private Map<Long, List<String>> partitionValuesMap;
@@ -1093,7 +1095,7 @@ public class HiveMetaStoreCache {
Map<Range<PartitionKey>, UniqueId> rangeToId,
RangeMap<ColumnBound, UniqueId> singleColumnRangeMap,
long nextPartitionId,
- Map<String, Long> partitionNameToIdMap,
+ BiMap<String, Long> partitionNameToIdMap,
Map<Long, List<UniqueId>> idToUniqueIdsMap,
Map<UniqueId, Range<ColumnBound>> singleUidToColumnRangeMap,
Map<Long, List<String>> partitionValuesMap) {
@@ -1111,7 +1113,7 @@ public class HiveMetaStoreCache {
public HivePartitionValues copy() {
HivePartitionValues copy = new HivePartitionValues();
copy.setNextPartitionId(nextPartitionId);
- copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null :
Maps.newHashMap(partitionNameToIdMap));
+ copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null :
HashBiMap.create(partitionNameToIdMap));
copy.setIdToUniqueIdsMap(idToUniqueIdsMap == null ? null :
Maps.newHashMap(idToUniqueIdsMap));
copy.setIdToPartitionItem(idToPartitionItem == null ? null :
Maps.newHashMap(idToPartitionItem));
copy.setPartitionValuesMap(partitionValuesMap == null ? null :
Maps.newHashMap(partitionValuesMap));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 61fc3e91651..b07ca6ad1d1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -41,7 +41,6 @@ import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -166,14 +165,6 @@ public class MTMVPartitionUtil {
return result.getDescs();
}
- public static List<String> getPartitionNamesByIds(MTMV mtmv,
Collection<Long> ids) throws AnalysisException {
- List<String> res = Lists.newArrayList();
- for (Long partitionId : ids) {
- res.add(mtmv.getPartitionName(partitionId));
- }
- return res;
- }
-
public static List<Long> getPartitionsIdsByNames(MTMV mtmv, List<String>
partitions) throws AnalysisException {
mtmv.readLock();
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
index 8aee7741cee..c34df750de5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java
@@ -79,15 +79,6 @@ public interface MTMVRelatedTableIf extends TableIf {
*/
MTMVSnapshotIf getTableSnapshot() throws AnalysisException;
- /**
- * getPartitionName
- *
- * @param partitionId
- * @return partitionName
- * @throws AnalysisException
- */
- String getPartitionName(long partitionId) throws AnalysisException;
-
/**
* Does the current type of table allow timed triggering
*
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
index 364e1f23b16..5f1e19f3284 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java
@@ -63,6 +63,8 @@ import org.apache.doris.utframe.TestWithFeService;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.base.Preconditions;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.HashBiMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Range;
@@ -477,6 +479,7 @@ public class CatalogMgrTest extends TestWithFeService {
partitionValueCacheKey.getTypes());
HivePartitionValues partitionValues =
metaStoreCache.getPartitionValues(partitionValueCacheKey);
Assert.assertEquals(partitionValues.getPartitionNameToIdMap().size(),
4);
+
Assert.assertEquals(partitionValues.getPartitionNameToIdMap().inverse().size(),
4);
}
@Test
@@ -520,7 +523,7 @@ public class CatalogMgrTest extends TestWithFeService {
HiveMetaStoreCache metaStoreCache) {
// partition name format: nation=cn/city=beijing
Map<Long, PartitionItem> idToPartitionItem =
Maps.newHashMapWithExpectedSize(partitionNames.size());
- Map<String, Long> partitionNameToIdMap =
Maps.newHashMapWithExpectedSize(partitionNames.size());
+ BiMap<String, Long> partitionNameToIdMap =
HashBiMap.create(partitionNames.size());
Map<Long, List<UniqueId>> idToUniqueIdsMap =
Maps.newHashMapWithExpectedSize(partitionNames.size());
long idx = 0;
for (String partitionName : partitionNames) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
index d40e6db5d44..d6c4a87f224 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java
@@ -99,10 +99,6 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = baseSnapshotIf;
- mtmv.getPartitionName(anyLong);
- minTimes = 0;
- result = "name1";
-
mtmv.getRefreshSnapshot();
minTimes = 0;
result = refreshSnapshot;
@@ -123,10 +119,6 @@ public class MTMVPartitionUtilTest {
minTimes = 0;
result = baseSnapshotIf;
- baseOlapTable.getPartitionName(anyLong);
- minTimes = 0;
- result = "name1";
-
refreshSnapshot.equalsWithRelatedPartition(anyString,
anyString, (MTMVSnapshotIf) any);
minTimes = 0;
result = true;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
