This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3bf00447fdf [fix](restore) Fix the conflict IDs between two cluster
(#33440)
3bf00447fdf is described below
commit 3bf00447fdff571fb5a19376fcc47cdac2740988
Author: walter <[email protected]>
AuthorDate: Wed Apr 10 21:56:31 2024 +0800
[fix](restore) Fix the conflict IDs between two cluster (#33440)
The meta of the restore may come from different clusters, so the
original ID in the meta may conflict with the ID of the new cluster. For
example, if a newly allocated ID happens to be the same as an original ID,
the original one may be overwritten when executing `put`.
---
.../java/org/apache/doris/catalog/OlapTable.java | 47 ++++++++++------------
.../org/apache/doris/catalog/PartitionInfo.java | 30 ++++++++++----
2 files changed, 45 insertions(+), 32 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 536b8032a99..cbeea28b378 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -542,7 +542,13 @@ public class OlapTable extends Table {
}
public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation
restoreReplicaAlloc,
- boolean reserveReplica) {
+ boolean reserveReplica) {
+ // ATTN: The meta of the restore may come from different clusters, so
the
+ // original ID in the meta may conflict with the ID of the new
cluster. For
+ // example, if a newly allocated ID happens to be the same as an
original ID,
+ // the original one may be overwritten when executing `put`, then
causes a
+ // NullPointerException.
+
// table id
id = env.getNextId();
@@ -553,13 +559,15 @@ public class OlapTable extends Table {
}
// reset all 'indexIdToXXX' map
+ Map<Long, MaterializedIndexMeta> origIdxIdToMeta = indexIdToMeta;
+ indexIdToMeta = Maps.newHashMap();
for (Map.Entry<Long, String> entry : origIdxIdToName.entrySet()) {
long newIdxId = env.getNextId();
if (entry.getValue().equals(name)) {
// base index
baseIndexId = newIdxId;
}
- indexIdToMeta.put(newIdxId, indexIdToMeta.remove(entry.getKey()));
+ indexIdToMeta.put(newIdxId, origIdxIdToMeta.get(entry.getKey()));
indexNameToId.put(entry.getValue(), newIdxId);
}
@@ -570,35 +578,24 @@ public class OlapTable extends Table {
}
// reset partition info and idToPartition map
- if (partitionInfo.getType() == PartitionType.RANGE ||
partitionInfo.getType() == PartitionType.LIST) {
- for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
- long newPartId = env.getNextId();
- if (reserveReplica) {
- ReplicaAllocation originReplicaAlloc =
partitionInfo.getReplicaAllocation(entry.getValue());
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), originReplicaAlloc, false);
- } else {
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, false);
- }
- idToPartition.put(newPartId,
idToPartition.remove(entry.getValue()));
- }
- } else {
- // Single partitioned
+ Map<Long, Long> partitionMap = Maps.newHashMap();
+ Map<Long, Partition> origIdToPartition = idToPartition;
+ idToPartition = Maps.newHashMap();
+ for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
long newPartId = env.getNextId();
- for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
- if (reserveReplica) {
- ReplicaAllocation originReplicaAlloc =
partitionInfo.getReplicaAllocation(entry.getValue());
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), originReplicaAlloc, true);
- } else {
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, true);
- }
- idToPartition.put(newPartId,
idToPartition.remove(entry.getValue()));
- }
+ idToPartition.put(newPartId,
origIdToPartition.get(entry.getValue()));
+ partitionMap.put(newPartId, entry.getValue());
}
+ boolean isSinglePartition = partitionInfo.getType() !=
PartitionType.RANGE
+ && partitionInfo.getType() != PartitionType.LIST;
+ partitionInfo.resetPartitionIdForRestore(partitionMap,
+ reserveReplica ? null : restoreReplicaAlloc,
isSinglePartition);
// for each partition, reset rollup index map
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
Partition partition = entry.getValue();
- // entry.getKey() is the new partition id, use it to get the
restore specified replica allocation
+ // entry.getKey() is the new partition id, use it to get the
restore specified
+ // replica allocation
ReplicaAllocation replicaAlloc =
partitionInfo.getReplicaAllocation(entry.getKey());
for (Map.Entry<Long, String> entry2 : origIdxIdToName.entrySet()) {
MaterializedIndex idx = partition.getIndex(entry2.getKey());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
index a8033b788f1..8cf1d664f52 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionInfo.java
@@ -349,15 +349,31 @@ public class PartitionInfo implements Writable {
}
}
- public void resetPartitionIdForRestore(long newPartitionId, long
oldPartitionId,
+ public void resetPartitionIdForRestore(
+ Map<Long, Long> partitionIdMap,
ReplicaAllocation restoreReplicaAlloc, boolean
isSinglePartitioned) {
- idToDataProperty.put(newPartitionId,
idToDataProperty.remove(oldPartitionId));
- idToReplicaAllocation.remove(oldPartitionId);
- idToReplicaAllocation.put(newPartitionId, restoreReplicaAlloc);
- if (!isSinglePartitioned) {
- idToItem.put(newPartitionId, idToItem.remove(oldPartitionId));
+ Map<Long, DataProperty> origIdToDataProperty = idToDataProperty;
+ Map<Long, ReplicaAllocation> origIdToReplicaAllocation =
idToReplicaAllocation;
+ Map<Long, PartitionItem> origIdToItem = idToItem;
+ Map<Long, Boolean> origIdToInMemory = idToInMemory;
+ Map<Long, String> origIdToStoragePolicy = idToStoragePolicy;
+ idToDataProperty = Maps.newHashMap();
+ idToReplicaAllocation = Maps.newHashMap();
+ idToItem = Maps.newHashMap();
+ idToInMemory = Maps.newHashMap();
+ idToStoragePolicy = Maps.newHashMap();
+
+ for (Map.Entry<Long, Long> entry : partitionIdMap.entrySet()) {
+ idToDataProperty.put(entry.getKey(),
origIdToDataProperty.get(entry.getValue()));
+ idToReplicaAllocation.put(entry.getKey(),
+ restoreReplicaAlloc == null ?
origIdToReplicaAllocation.get(entry.getValue())
+ : restoreReplicaAlloc);
+ if (!isSinglePartitioned) {
+ idToItem.put(entry.getKey(),
origIdToItem.get(entry.getValue()));
+ }
+ idToInMemory.put(entry.getKey(),
origIdToInMemory.get(entry.getValue()));
+ idToStoragePolicy.put(entry.getKey(),
origIdToStoragePolicy.get(entry.getValue()));
}
- idToInMemory.put(newPartitionId, idToInMemory.remove(oldPartitionId));
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
