This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit fe772c76e7d8d0309838be18d8c0382c330dd4af Author: walter <[email protected]> AuthorDate: Wed Apr 10 18:57:10 2024 +0800 [fix](restore) Fix the conflict IDs between two cluster (#33423) The meta of the restore may come from different clusters, so the original ID in the meta may conflict with the ID of the new cluster. For example, if a newly allocated ID happens to be the same as an original ID, the original one may be overwritten when executing `put`. --- .../java/org/apache/doris/catalog/OlapTable.java | 51 ++++++++++------------ .../org/apache/doris/catalog/PartitionInfo.java | 30 ++++++++++--- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 723f7dba065..c79a71461b8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -566,7 +566,13 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { } public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restoreReplicaAlloc, - boolean reserveReplica) { + boolean reserveReplica) { + // ATTN: The meta of the restore may come from different clusters, so the + // original ID in the meta may conflict with the ID of the new cluster. For + // example, if a newly allocated ID happens to be the same as an original ID, + // the original one may be overwritten when executing `put`, then causes a + // NullPointerException. + // table id id = env.getNextId(); @@ -577,13 +583,15 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { } // reset all 'indexIdToXXX' map + Map<Long, MaterializedIndexMeta> origIdxIdToMeta = indexIdToMeta; + indexIdToMeta = Maps.newHashMap(); for (Map.Entry<Long, String> entry : origIdxIdToName.entrySet()) { long newIdxId = env.getNextId(); if (entry.getValue().equals(name)) { // base index baseIndexId = newIdxId; } - indexIdToMeta.put(newIdxId, indexIdToMeta.remove(entry.getKey())); + indexIdToMeta.put(newIdxId, origIdxIdToMeta.get(entry.getKey())); indexNameToId.put(entry.getValue(), newIdxId); } @@ -594,36 +602,25 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { } // reset partition info and idToPartition map - if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { - for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) { - long newPartId = env.getNextId(); - if (reserveReplica) { - ReplicaAllocation originReplicaAlloc = partitionInfo.getReplicaAllocation(entry.getValue()); - partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), originReplicaAlloc, false); - } else { - partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), restoreReplicaAlloc, false); - } - idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); - } - } else { - // Single partitioned + Map<Long, Long> partitionMap = Maps.newHashMap(); + Map<Long, Partition> origIdToPartition = idToPartition; + idToPartition = Maps.newHashMap(); + for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) { long newPartId = env.getNextId(); - for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) { - if (reserveReplica) { - ReplicaAllocation originReplicaAlloc = partitionInfo.getReplicaAllocation(entry.getValue()); - partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), originReplicaAlloc, true); - } else { - partitionInfo.resetPartitionIdForRestore(newPartId, entry.getValue(), restoreReplicaAlloc, true); - } - idToPartition.put(newPartId, idToPartition.remove(entry.getValue())); - } + idToPartition.put(newPartId, origIdToPartition.get(entry.getValue())); + partitionMap.put(newPartId, entry.getValue()); } + boolean isSinglePartition = partitionInfo.getType() != PartitionType.RANGE + && partitionInfo.getType() != PartitionType.LIST; + partitionInfo.resetPartitionIdForRestore(partitionMap, + reserveReplica ? null : restoreReplicaAlloc, isSinglePartition); // for each partition, reset rollup index map - Map<Tag, Integer> nextIndexs = Maps.newHashMap(); + Map<Tag, Integer> nextIndexes = Maps.newHashMap(); for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) { Partition partition = entry.getValue(); - // entry.getKey() is the new partition id, use it to get the restore specified replica allocation + // entry.getKey() is the new partition id, use it to get the restore specified + // replica allocation ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(entry.getKey()); for (Map.Entry<Long, String> entry2 : origIdxIdToName.entrySet()) { MaterializedIndex idx = partition.getIndex(entry2.getKey()); @@ -648,7 +645,7 @@ public class OlapTable extends Table implements MTMVRelatedTableIf { try { Pair<Map<Tag, List<Long>>, TStorageMedium> tag2beIdsAndMedium = Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation( - replicaAlloc, nextIndexs, null, false, false); + replicaAlloc, nextIndexes, null, false, false); Map<Tag, List<Long>> tag2beIds = tag2beIdsAndMedium.first; for (Map.Entry<Tag, List<Long>> entry3 : tag2beIds.entrySet()) { for (Long beId : entry3.getValue()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java index 70aee07fad2..c899a4e8917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java @@ -390,15 +390,31 @@ public class PartitionInfo implements Writable { } } - public void resetPartitionIdForRestore(long newPartitionId, long oldPartitionId, + public void resetPartitionIdForRestore( + Map<Long, Long> partitionIdMap, ReplicaAllocation restoreReplicaAlloc, boolean isSinglePartitioned) { - idToDataProperty.put(newPartitionId, idToDataProperty.remove(oldPartitionId)); - idToReplicaAllocation.remove(oldPartitionId); - idToReplicaAllocation.put(newPartitionId, restoreReplicaAlloc); - if (!isSinglePartitioned) { - idToItem.put(newPartitionId, idToItem.remove(oldPartitionId)); + Map<Long, DataProperty> origIdToDataProperty = idToDataProperty; + Map<Long, ReplicaAllocation> origIdToReplicaAllocation = idToReplicaAllocation; + Map<Long, PartitionItem> origIdToItem = idToItem; + Map<Long, Boolean> origIdToInMemory = idToInMemory; + Map<Long, String> origIdToStoragePolicy = idToStoragePolicy; + idToDataProperty = Maps.newHashMap(); + idToReplicaAllocation = Maps.newHashMap(); + idToItem = Maps.newHashMap(); + idToInMemory = Maps.newHashMap(); + idToStoragePolicy = Maps.newHashMap(); + + for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) { + idToDataProperty.put(entry.getKey(), origIdToDataProperty.get(entry.getValue())); + idToReplicaAllocation.put(entry.getKey(), + restoreReplicaAlloc == null ? origIdToReplicaAllocation.get(entry.getValue()) + : restoreReplicaAlloc); + if (!isSinglePartitioned) { + idToItem.put(entry.getKey(), origIdToItem.get(entry.getValue())); + } + idToInMemory.put(entry.getKey(), origIdToInMemory.get(entry.getValue())); + idToStoragePolicy.put(entry.getKey(), origIdToStoragePolicy.get(entry.getValue())); } - idToInMemory.put(newPartitionId, idToInMemory.remove(oldPartitionId)); } @Override --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
