This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 53b79d5a8c [Enhancement](restore) new add the property of
reserve_replica to restore statement (#11942)
53b79d5a8c is described below
commit 53b79d5a8c6981c11e7b16ac7432dd357f2d6fc9
Author: xueweizhang <[email protected]>
AuthorDate: Tue Sep 6 10:32:21 2022 +0800
[Enhancement](restore) new add the property of reserve_replica to restore
statement (#11942)
Add a new property called 'reserve_replica', which means you can
get a table with same partitions with the same replication num
as before the backup.
Co-authored-by: Stalary <[email protected]>
Co-authored-by: camby <[email protected]>
---
.../Backup-and-Restore/RESTORE.md | 1 +
.../Backup-and-Restore/RESTORE.md | 1 +
.../org/apache/doris/analysis/RestoreStmt.java | 18 ++++++++
.../org/apache/doris/analysis/ShowRestoreStmt.java | 2 +-
.../org/apache/doris/backup/BackupHandler.java | 2 +-
.../java/org/apache/doris/backup/RestoreJob.java | 51 ++++++++++++++++++----
.../java/org/apache/doris/catalog/OlapTable.java | 17 ++++++--
.../org/apache/doris/backup/RestoreJobTest.java | 2 +-
8 files changed, 79 insertions(+), 15 deletions(-)
diff --git
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
index 5d54c6c24a..5ed19eb5d7 100644
---
a/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
+++
b/docs/en/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
@@ -57,6 +57,7 @@ illustrate:
- PROPERTIES currently supports the following properties:
- "backup_timestamp" = "2018-05-04-16-45-08": Specifies which time version
of the corresponding backup to restore, required. This information can be
obtained with the `SHOW SNAPSHOT ON repo;` statement.
- "replication_num" = "3": Specifies the number of replicas for the restored
table or partition. Default is 3. If restoring an existing table or partition,
the number of replicas must be the same as the number of replicas of the
existing table or partition. At the same time, there must be enough hosts to
accommodate multiple replicas.
+ - "reserve_replica" = "true": Default is false. When this property is true,
the replication_num property is ignored and the restored table or partition
will have the same number of replication as before the backup. Supports
multiple tables or multiple partitions within a table with different
replication number.
- "timeout" = "3600": The task timeout period, the default is one day. in
seconds.
- "meta_version" = 40: Use the specified meta_version to read the previously
backed up metadata. Note that this parameter is used as a temporary solution
and is only used to restore the data backed up by the old version of Doris. The
latest version of the backup data already contains the meta version, no need to
specify it.
diff --git
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
index c9ef66be9a..2f912c6b92 100644
---
a/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
+++
b/docs/zh-CN/docs/sql-manual/sql-reference/Data-Definition-Statements/Backup-and-Restore/RESTORE.md
@@ -56,6 +56,7 @@ PROPERTIES ("key"="value", ...);
- PROPERTIES 目前支持以下属性：
- "backup_timestamp" = "2018-05-04-16-45-08"：指定了恢复对应备份的哪个时间版本，必填。该信息可以通过
`SHOW SNAPSHOT ON repo;` 语句获得。
- "replication_num" =
"3"：指定恢复的表或分区的副本数。默认为3。若恢复已存在的表或分区，则副本数必须和已存在表或分区的副本数相同。同时，必须有足够的 host 容纳多个副本。
+ - "reserve_replica" = "true"：默认为 false。当该属性为 true 时，会忽略 replication_num
属性，恢复的表或分区的副本数将与备份之前一样。支持多个表或表内多个分区有不同的副本数。
- "timeout" = "3600"：任务超时时间，默认为一天。单位秒。
- "meta_version" = 40：使用指定的 meta_version 来读取之前备份的元数据。注意，该参数作为临时方案，仅用于恢复老版本
Doris 备份的数据。最新版本的备份数据中已经包含 meta version，无需再指定。
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
index cb9b4e3bb4..a98e6f2891 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java
@@ -36,11 +36,13 @@ public class RestoreStmt extends AbstractBackupStmt {
private static final String PROP_REPLICATION_NUM = "replication_num";
private static final String PROP_BACKUP_TIMESTAMP = "backup_timestamp";
private static final String PROP_META_VERSION = "meta_version";
+ private static final String PROP_RESERVE_REPLICA = "reserve_replica";
private boolean allowLoad = false;
private ReplicaAllocation replicaAlloc =
ReplicaAllocation.DEFAULT_ALLOCATION;
private String backupTimestamp = null;
private int metaVersion = -1;
+ private boolean reserveReplica = false;
public RestoreStmt(LabelName labelName, String repoName,
AbstractBackupTableRefClause restoreTableRefClause,
Map<String, String> properties) {
@@ -63,6 +65,10 @@ public class RestoreStmt extends AbstractBackupStmt {
return metaVersion;
}
+ public boolean reserveReplica() {
+ return reserveReplica;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
@@ -106,6 +112,18 @@ public class RestoreStmt extends AbstractBackupStmt {
if (this.replicaAlloc.isNotSet()) {
this.replicaAlloc = ReplicaAllocation.DEFAULT_ALLOCATION;
}
+ // reserve replica
+ if (copiedProperties.containsKey(PROP_RESERVE_REPLICA)) {
+ if
(copiedProperties.get(PROP_RESERVE_REPLICA).equalsIgnoreCase("true")) {
+ reserveReplica = true;
+ } else if
(copiedProperties.get(PROP_RESERVE_REPLICA).equalsIgnoreCase("false")) {
+ reserveReplica = false;
+ } else {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid reserve_replica value: " +
copiedProperties.get(PROP_RESERVE_REPLICA));
+ }
+ copiedProperties.remove(PROP_RESERVE_REPLICA);
+ }
// backup timestamp
if (copiedProperties.containsKey(PROP_BACKUP_TIMESTAMP)) {
backupTimestamp = copiedProperties.get(PROP_BACKUP_TIMESTAMP);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
index 8d17532aa1..43f703fb75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowRestoreStmt.java
@@ -39,7 +39,7 @@ import java.util.function.Predicate;
public class ShowRestoreStmt extends ShowStmt {
public static final ImmutableList<String> TITLE_NAMES = new
ImmutableList.Builder<String>()
.add("JobId").add("Label").add("Timestamp").add("DbName").add("State")
- .add("AllowLoad").add("ReplicationNum").add("ReplicaAllocation")
+
.add("AllowLoad").add("ReplicationNum").add("ReplicaAllocation").add("ReserveReplica")
.add("RestoreObjs").add("CreateTime").add("MetaPreparedTime").add("SnapshotFinishedTime")
.add("DownloadFinishedTime").add("FinishedTime").add("UnfinishedTasks").add("Progress")
.add("TaskErrMsg").add("Status").add("Timeout")
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 1329f64b48..fd1e8efdd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -401,7 +401,7 @@ public class BackupHandler extends MasterDaemon implements
Writable {
// Create a restore job
RestoreJob restoreJob = new RestoreJob(stmt.getLabel(),
stmt.getBackupTimestamp(),
db.getId(), db.getFullName(), jobInfo, stmt.allowLoad(),
stmt.getReplicaAlloc(),
- stmt.getTimeoutMs(), stmt.getMetaVersion(), env,
repository.getId());
+ stmt.getTimeoutMs(), stmt.getMetaVersion(),
stmt.reserveReplica(), env, repository.getId());
env.getEditLog().logRestoreJob(restoreJob);
// must put to dbIdToBackupOrRestoreJob after edit log, otherwise the
state of job may be changed.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 4a592d71cf..ceeaeebfce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -97,6 +97,8 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
public class RestoreJob extends AbstractJob {
+ private static final String PROP_RESERVE_REPLICA = "reserve_replica";
+
private static final Logger LOG = LogManager.getLogger(RestoreJob.class);
// CHECKSTYLE OFF
@@ -132,6 +134,8 @@ public class RestoreJob extends AbstractJob {
private ReplicaAllocation replicaAlloc;
+ private boolean reserveReplica = false;
+
// this 2 members is to save all newly restored objs
// tbl name -> part
private List<Pair<String, Partition>> restoredPartitions =
Lists.newArrayList();
@@ -162,7 +166,8 @@ public class RestoreJob extends AbstractJob {
}
public RestoreJob(String label, String backupTs, long dbId, String dbName,
BackupJobInfo jobInfo, boolean allowLoad,
- ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion,
Env env, long repoId) {
+ ReplicaAllocation replicaAlloc, long timeoutMs, int metaVersion,
boolean reserveReplica, Env env,
+ long repoId) {
super(JobType.RESTORE, label, dbId, dbName, timeoutMs, env, repoId);
this.backupTimestamp = backupTs;
this.jobInfo = jobInfo;
@@ -170,6 +175,8 @@ public class RestoreJob extends AbstractJob {
this.replicaAlloc = replicaAlloc;
this.state = RestoreJobState.PENDING;
this.metaVersion = metaVersion;
+ this.reserveReplica = reserveReplica;
+ properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica));
}
public RestoreJobState getState() {
@@ -566,9 +573,13 @@ public class RestoreJob extends AbstractJob {
String partitionName = partitionEntry.getKey();
BackupPartitionInfo backupPartInfo =
partitionEntry.getValue();
Partition localPartition =
localOlapTbl.getPartition(partitionName);
+ Partition remotePartition =
remoteOlapTbl.getPartition(partitionName);
if (localPartition != null) {
// Partition already exist.
PartitionInfo localPartInfo =
localOlapTbl.getPartitionInfo();
+ PartitionInfo remotePartInfo =
remoteOlapTbl.getPartitionInfo();
+ ReplicaAllocation remoteReplicaAlloc =
remotePartInfo.getReplicaAllocation(
+ remotePartition.getId());
if (localPartInfo.getType() ==
PartitionType.RANGE
|| localPartInfo.getType() ==
PartitionType.LIST) {
PartitionItem localItem =
localPartInfo.getItem(localPartition.getId());
@@ -577,7 +588,7 @@ public class RestoreJob extends AbstractJob {
if (localItem.equals(remoteItem)) {
// Same partition, same range
if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition,
- localTbl, backupPartInfo,
partitionName, tblInfo)) {
+ localTbl, backupPartInfo,
partitionName, tblInfo, remoteReplicaAlloc)) {
return;
}
} else {
@@ -590,7 +601,7 @@ public class RestoreJob extends AbstractJob {
} else {
// If this is a single partitioned table.
if
(genFileMappingWhenBackupReplicasEqual(localPartInfo, localPartition, localTbl,
- backupPartInfo, partitionName,
tblInfo)) {
+ backupPartInfo, partitionName,
tblInfo, remoteReplicaAlloc)) {
return;
}
}
@@ -609,10 +620,16 @@ public class RestoreJob extends AbstractJob {
return;
} else {
// this partition can be added to this
table, set ids
+ ReplicaAllocation restoreReplicaAlloc
= replicaAlloc;
+ if (reserveReplica) {
+ PartitionInfo remotePartInfo =
remoteOlapTbl.getPartitionInfo();
+ restoreReplicaAlloc =
remotePartInfo.getReplicaAllocation(
+ remotePartition.getId());
+ }
Partition restorePart =
resetPartitionForRestore(localOlapTbl, remoteOlapTbl,
partitionName,
db.getClusterName(),
- replicaAlloc);
+ restoreReplicaAlloc);
if (restorePart == null) {
return;
}
@@ -642,7 +659,7 @@ public class RestoreJob extends AbstractJob {
}
// reset all ids in this table
- Status st = remoteOlapTbl.resetIdsForRestore(env, db,
replicaAlloc);
+ Status st = remoteOlapTbl.resetIdsForRestore(env, db,
replicaAlloc, reserveReplica);
if (!st.ok()) {
status = st;
return;
@@ -794,8 +811,12 @@ public class RestoreJob extends AbstractJob {
long remotePartId = backupPartitionInfo.id;
PartitionItem remoteItem =
remoteTbl.getPartitionInfo().getItem(remotePartId);
DataProperty remoteDataProperty =
remotePartitionInfo.getDataProperty(remotePartId);
+ ReplicaAllocation restoreReplicaAlloc = replicaAlloc;
+ if (reserveReplica) {
+ restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
+ }
localPartitionInfo.addPartition(restoredPart.getId(),
false, remoteItem,
- remoteDataProperty, replicaAlloc,
+ remoteDataProperty, restoreReplicaAlloc,
remotePartitionInfo.getIsInMemory(remotePartId));
}
localTbl.addPartition(restoredPart);
@@ -927,9 +948,15 @@ public class RestoreJob extends AbstractJob {
}
private boolean genFileMappingWhenBackupReplicasEqual(PartitionInfo
localPartInfo, Partition localPartition,
- Table localTbl, BackupPartitionInfo backupPartInfo, String
partitionName, BackupOlapTableInfo tblInfo) {
- short restoreReplicaNum = replicaAlloc.getTotalReplicaNum();
+ Table localTbl, BackupPartitionInfo backupPartInfo, String
partitionName, BackupOlapTableInfo tblInfo,
+ ReplicaAllocation remoteReplicaAlloc) {
+ short restoreReplicaNum;
short localReplicaNum =
localPartInfo.getReplicaAllocation(localPartition.getId()).getTotalReplicaNum();
+ if (!reserveReplica) {
+ restoreReplicaNum = replicaAlloc.getTotalReplicaNum();
+ } else {
+ restoreReplicaNum = remoteReplicaAlloc.getTotalReplicaNum();
+ }
if (localReplicaNum != restoreReplicaNum) {
status = new Status(ErrCode.COMMON_ERROR, "Partition " +
partitionName
+ " in table " + localTbl.getName()
@@ -1127,8 +1154,12 @@ public class RestoreJob extends AbstractJob {
.getPartInfo(restorePart.getName());
long remotePartId = backupPartitionInfo.id;
DataProperty remoteDataProperty =
remotePartitionInfo.getDataProperty(remotePartId);
+ ReplicaAllocation restoreReplicaAlloc = replicaAlloc;
+ if (reserveReplica) {
+ restoreReplicaAlloc =
remotePartitionInfo.getReplicaAllocation(remotePartId);
+ }
localPartitionInfo.addPartition(restorePart.getId(), false,
remotePartitionInfo.getItem(remotePartId),
- remoteDataProperty, replicaAlloc,
+ remoteDataProperty, restoreReplicaAlloc,
remotePartitionInfo.getIsInMemory(remotePartId));
localTbl.addPartition(restorePart);
@@ -1514,6 +1545,7 @@ public class RestoreJob extends AbstractJob {
info.add(String.valueOf(allowLoad));
info.add(String.valueOf(replicaAlloc.getTotalReplicaNum()));
info.add(replicaAlloc.toCreateStmt());
+ info.add(String.valueOf(reserveReplica));
info.add(getRestoreObjs());
info.add(TimeUtils.longToTimeString(createTime));
info.add(TimeUtils.longToTimeString(metaPreparedTime));
@@ -1858,6 +1890,7 @@ public class RestoreJob extends AbstractJob {
String value = Text.readString(in);
properties.put(key, value);
}
+ reserveReplica =
Boolean.parseBoolean(properties.get(PROP_RESERVE_REPLICA));
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 81163e68fc..61a835ee41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -433,7 +433,8 @@ public class OlapTable extends Table {
setColocateGroup(null);
}
- public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation
restoreReplicaAlloc) {
+ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation
restoreReplicaAlloc,
+ boolean reserveReplica) {
// table id
id = env.getNextId();
@@ -464,14 +465,24 @@ public class OlapTable extends Table {
if (partitionInfo.getType() == PartitionType.RANGE ||
partitionInfo.getType() == PartitionType.LIST) {
for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
long newPartId = env.getNextId();
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, false);
+ if (reserveReplica) {
+ ReplicaAllocation originReplicaAlloc =
partitionInfo.getReplicaAllocation(entry.getValue());
+ partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), originReplicaAlloc, false);
+ } else {
+ partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, false);
+ }
idToPartition.put(newPartId,
idToPartition.remove(entry.getValue()));
}
} else {
// Single partitioned
long newPartId = env.getNextId();
for (Map.Entry<String, Long> entry : origPartNameToId.entrySet()) {
- partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, true);
+ if (reserveReplica) {
+ ReplicaAllocation originReplicaAlloc =
partitionInfo.getReplicaAllocation(entry.getValue());
+ partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), originReplicaAlloc, true);
+ } else {
+ partitionInfo.resetPartitionIdForRestore(newPartId,
entry.getValue(), restoreReplicaAlloc, true);
+ }
idToPartition.put(newPartId,
idToPartition.remove(entry.getValue()));
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index bc16a865c6..81930d5c73 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -242,7 +242,7 @@ public class RestoreJobTest {
db.dropTable(expectedRestoreTbl.getName());
job = new RestoreJob(label, "2018-01-01 01:01:01", db.getId(),
db.getFullName(), jobInfo, false,
- new ReplicaAllocation((short) 3), 100000, -1, env,
repo.getId());
+ new ReplicaAllocation((short) 3), 100000, -1, false, env,
repo.getId());
List<Table> tbls = Lists.newArrayList();
List<Resource> resources = Lists.newArrayList();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
